From 5434504f9bf3d2f6ad79b9149a55d309830b8608 Mon Sep 17 00:00:00 2001 From: panyy Date: Thu, 5 Nov 2026 18:16:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E4=BC=98=E5=8C=96=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E4=BD=93=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../step/chat_step/impl/base_chat_step.py | 35 +- apps/application/flow/step_node/__init__.py | 3 +- .../ai_chat_step_node/impl/base_chat_node.py | 38 +- .../flow/step_node/mcp_node/i_mcp_node.py | 6 +- .../step_node/mcp_node/impl/base_mcp_node.py | 104 ++--- apps/application/flow/tools.py | 42 +- apps/application/serializers/application.py | 19 +- apps/application/views/application_chat.py | 2 +- apps/common/config/tokenizer_manage_config.py | 39 +- .../handle/impl/text/xlsx_split_handle.py | 2 +- apps/knowledge/embedding_params.py | 3 +- apps/knowledge/serializers/document.py | 4 +- apps/knowledge/serializers/knowledge.py | 25 +- apps/knowledge/serializers/knowledge_ext.py | 2 +- apps/knowledge/test_embedding_params.py | 18 +- apps/knowledge/test_knowledge_ext.py | 5 + apps/models_provider/base_model_provider.py | 3 + .../model/llm.py | 5 + .../impl/ollama_model_provider/model/llm.py | 2 + apps/models_provider/local_imports.py | 101 +++- apps/models_provider/test_local_imports.py | 49 ++ apps/models_provider/tools.py | 12 +- apps/system_manage/ai_hub.py | 165 +++++-- apps/system_manage/ai_hub_assets.py | 4 +- apps/system_manage/ai_hub_organize.py | 400 +++++++++++++--- apps/system_manage/ai_hub_progress.py | 3 +- apps/system_manage/ai_hub_worker.py | 145 +++--- .../management/commands/ai_hub_worker.py | 70 ++- apps/system_manage/parsers/native.py | 43 +- apps/system_manage/test_ai_hub.py | 77 +++ apps/system_manage/test_ai_hub_organize.py | 310 ++++++++++++- apps/system_manage/urls.py | 5 +- apps/system_manage/views/ai_hub.py | 186 +++++++- scripts/ai_hub_ops.py | 3 +- ui/src/api/ai-hub.ts | 3 +- .../ExecutionDetailCard.vue | 8 + ui/src/enums/application.ts | 1 + ui/src/layout/components/sidebar/index.vue | 60 ++- ui/src/layout/layout-header/avatar/index.vue | 2 +- ui/src/router/modules/system.ts | 15 +- ui/src/views/application-workflow/index.vue | 27 ++ .../component/AIModeParamSettingDialog.vue | 15 +- ui/src/views/document/upload/SetRules.vue | 4 +- ui/src/views/knowledge/KnowledgeSetting.vue | 48 +- ui/src/views/knowledge/component/BaseForm.vue | 2 + ui/src/views/mindmap/HubManager.vue | 438 +++++++++++++++--- .../views/mindmap/workbench/api/document.ts | 66 ++- .../workbench/components/ConfigPanel.vue | 111 ++++- .../workbench/components/MarkdownRenderer.vue | 71 ++- .../workbench/components/MindMapRenderer.vue | 48 +- .../composables/useDocumentProcessor.ts | 126 +++-- .../mindmap/workbench/utils/graphModel.ts | 4 +- .../mindmap/workbench/utils/imageAssets.ts | 30 +- .../workbench/views/DocumentProcessor.vue | 307 ++++++++++-- .../views/model/component/LocalModelFiles.vue | 2 +- ui/src/views/model/component/ModelCard.vue | 1 + ui/src/views/system-setting/policy/index.vue | 36 +- ui/src/views/system/mindmap/index.vue | 33 +- ui/src/views/system/role/PlatformAccess.vue | 36 +- ui/src/views/system/user-manage/index.vue | 9 +- ui/src/workflow/common/data.ts | 15 +- ui/src/workflow/nodes/ai-chat-node/index.vue | 6 + ui/src/workflow/nodes/mcp-node/index.vue | 46 +- 63 files changed, 2815 insertions(+), 685 deletions(-) diff --git a/apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py b/apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py index cd17d4a10..bc20c415b 100644 --- a/apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py +++ b/apps/application/chat_pipeline/step/chat_step/impl/base_chat_step.py @@ -183,6 +183,10 @@ class BaseChatStep(IChatStep): tool_ids=None, mcp_output_enable=True, **kwargs): + from system_manage.mcp_agent import context_for + enabled = (model_setting or {}).get('engine_tools', []) + self.context['engine_body'] = {**manage.context, 'workspace_id': workspace_id, 'chat_id': str(chat_id)} + self.context['engine_tools'] = enabled chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, **model_params_setting) if model_id is not None else None if stream: @@ -208,7 +212,9 @@ class BaseChatStep(IChatStep): code_path = f'{executor.sandbox_path}/execute/{tool_id}.py' if os.path.exists(code_path): os.remove(code_path) + from system_manage.mcp_agent import call_records return { + 'engine_calls': call_records(self.context.get('engine_body', {})), 'step_type': 'chat_step', 'run_time': self.context['run_time'], 'model_id': str(manage.context['model_id']), @@ -231,25 +237,15 @@ class BaseChatStep(IChatStep): def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_ids, tool_ids, mcp_output_enable, chat_model, message_list): - if not mcp_enable and not tool_enable: + if not mcp_enable and not tool_enable and not self.context.get('engine_tools'): return None mcp_servers_config = {} - # 迁移过来mcp_source是None - if mcp_source is None: - mcp_source = 'custom' - if mcp_enable: - # 兼容老数据 - if not mcp_tool_ids: - mcp_tool_ids = [] - if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers: - mcp_servers_config = json.loads(mcp_servers) - elif mcp_tool_ids: - mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values() - for mcp_tool in mcp_tools: - if mcp_tool and mcp_tool['is_active']: - mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])} + from system_manage.mcp_agent import context_for + engine_context = context_for(self.context['engine_body'], engine_tools=self.context.get('engine_tools'), + tool_ids=mcp_tool_ids if mcp_enable and mcp_source not in (None, 'custom') else [], + custom=mcp_servers if mcp_enable and mcp_source in (None, 'custom') else None) if tool_enable: if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP @@ -269,8 +265,8 @@ class BaseChatStep(IChatStep): self.context['execute_ids'].append(_id) mcp_servers_config[str(tool.id)] = tool_config - if len(mcp_servers_config) > 0: - return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable) + if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or engine_context['engine_tools']: + return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context) return None @@ -332,6 +328,7 @@ class BaseChatStep(IChatStep): no_references_setting, problem_text, mcp_enable, mcp_tool_ids, mcp_servers, mcp_source, tool_enable, tool_ids, mcp_output_enable) chat_record_id = uuid.uuid7() + self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id) r = StreamingHttpResponse( streaming_content=event_content(chat_result, chat_id, chat_record_id, paragraph_list, post_response_handler, manage, self, chat_model, message_list, problem_text, @@ -375,7 +372,8 @@ class BaseChatStep(IChatStep): chat_model, message_list, ) if mcp_result: - return mcp_result, True + from application.flow.tools import collect_mcp_response + return collect_mcp_response(mcp_result), True return chat_model.invoke(message_list), True def execute_block(self, message_list: List[BaseMessage], @@ -401,6 +399,7 @@ class BaseChatStep(IChatStep): reasoning = Reasoning(reasoning_content_start, reasoning_content_end) chat_record_id = uuid.uuid7() + self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id) # 调用模型 try: chat_result, is_ai_chat = self.get_block_result(message_list, chat_model, paragraph_list, diff --git a/apps/application/flow/step_node/__init__.py b/apps/application/flow/step_node/__init__.py index ffb0c7828..99ae7fe48 100644 --- a/apps/application/flow/step_node/__init__.py +++ b/apps/application/flow/step_node/__init__.py @@ -21,6 +21,7 @@ from .loop_continue_node import BaseLoopContinueNode from .loop_node import * from .loop_start_node import * from .mcp_node import BaseMcpNode +from .component_node import BaseComponentNode from .parameter_extraction_node import BaseParameterExtractionNode from .question_node import * from .reranker_node import * @@ -42,7 +43,7 @@ node_list = [BaseStartStepNode, BaseChatNode, BaseSearchKnowledgeNode, BaseSearc BaseToolNodeNode, BaseToolLibNodeNode, BaseRerankerNode, BaseApplicationNode, BaseDocumentExtractNode, BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode, - BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseTextToVideoNode, BaseImageToVideoNode, + BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseComponentNode, BaseTextToVideoNode, BaseImageToVideoNode, BaseVideoUnderstandNode, BaseIntentNode, BaseLoopNode, BaseLoopStartStepNode, BaseLoopContinueNode, diff --git a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py b/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py index 20dc22b25..0cc086e96 100644 --- a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py +++ b/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py @@ -206,29 +206,23 @@ class BaseChatNode(IChatNode): def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_id, mcp_tool_ids, tool_ids, mcp_output_enable, chat_model, message_list, history_message, question): - if not mcp_enable and not tool_enable: + enabled = (self.node_params.get('model_setting') or {}).get('engine_tools', []) + if not mcp_enable and not tool_enable and not enabled: return None mcp_servers_config = {} - # 迁移过来mcp_source是None - if mcp_source is None: - mcp_source = 'custom' - if mcp_enable: - # 兼容老数据 - if not mcp_tool_ids: - mcp_tool_ids = [] - if mcp_tool_id: - mcp_tool_ids = list(set(mcp_tool_ids + [mcp_tool_id])) - if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers: - mcp_servers_config = json.loads(mcp_servers) - mcp_servers_config = self.handle_variables(mcp_servers_config) - elif mcp_tool_ids: - mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values() - for mcp_tool in mcp_tools: - if mcp_tool and mcp_tool['is_active']: - mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])} - mcp_servers_config = self.handle_variables(mcp_servers_config) + from system_manage.mcp_agent import context_for + ids = list(mcp_tool_ids or []) + if mcp_tool_id: ids.append(mcp_tool_id) + body = dict(self.workflow_manage.get_body()) + # Explicit AI-node selection is limited to knowledge bases associated with this workflow. + from system_manage.mcp_workflows import knowledge_scope + body['knowledge_id_list'] = knowledge_scope(self.workflow_manage.flow) + engine_context = context_for(body, engine_tools=enabled, + tool_ids=ids if mcp_enable and mcp_source not in (None, 'custom') else [], + custom=self.handle_variables(json.loads(mcp_servers) if isinstance(mcp_servers, str) else mcp_servers) if mcp_enable and mcp_source in (None, 'custom') and mcp_servers else None, + transform=self.handle_variables, node_id=self.id) if tool_enable: if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP @@ -248,8 +242,8 @@ class BaseChatNode(IChatNode): self.context['execute_ids'].append(_id) mcp_servers_config[str(tool.id)] = tool_config - if len(mcp_servers_config) > 0: - r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable) + if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or enabled: + r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context) return NodeResult( {'result': r, 'chat_model': chat_model, 'message_list': message_list, 'history_message': [{'content': message.content, 'role': message.type} for message in @@ -315,7 +309,9 @@ class BaseChatNode(IChatNode): code_path = f'{executor.sandbox_path}/execute/{tool_id}.py' if os.path.exists(code_path): os.remove(code_path) + from system_manage.mcp_agent import call_records return { + 'engine_calls': call_records(self.workflow_manage.get_body(), self.id), 'name': self.node.properties.get('stepName'), "index": index, 'run_time': self.context.get('run_time'), diff --git a/apps/application/flow/step_node/mcp_node/i_mcp_node.py b/apps/application/flow/step_node/mcp_node/i_mcp_node.py index cdd3f7e6a..d9ce9545e 100644 --- a/apps/application/flow/step_node/mcp_node/i_mcp_node.py +++ b/apps/application/flow/step_node/mcp_node/i_mcp_node.py @@ -9,12 +9,16 @@ from application.flow.i_step_node import INode, NodeResult class McpNodeSerializer(serializers.Serializer): - mcp_servers = serializers.JSONField(required=True, label=_("Mcp servers")) + mcp_servers = serializers.JSONField(required=False, default=dict, label=_("Mcp servers")) mcp_server = serializers.CharField(required=True, label=_("Mcp server")) mcp_tool = serializers.CharField(required=True, label=_("Mcp tool")) mcp_tool_id = serializers.CharField(required=False, label=_("Mcp tool"), allow_null=True, allow_blank=True) mcp_source = serializers.CharField(required=False, label=_("Mcp source"), allow_blank=True, allow_null=True) tool_params = serializers.DictField(required=True, label=_("Tool parameters")) + tool_form_field = serializers.ListField(required=False, default=list) + timeout = serializers.IntegerField(default=60, min_value=1, max_value=600) + retries = serializers.IntegerField(default=0, min_value=0, max_value=2) + priority = serializers.IntegerField(default=5, min_value=0, max_value=9) class IMcpNode(INode): diff --git a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py b/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py index 8fed9983e..e9c7c52d9 100644 --- a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py +++ b/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py @@ -1,75 +1,59 @@ # coding=utf-8 -import asyncio import json +from copy import deepcopy from typing import List - -from django.db.models import QuerySet -from langchain_mcp_adapters.client import MultiServerMCPClient - +from rest_framework.exceptions import ValidationError from application.flow.i_step_node import NodeResult from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode -from tools.models import Tool +from application.flow.step_node.component_node.base_component_node import workflow_identity +from system_manage import mcp_runtime class BaseMcpNode(IMcpNode): def save_context(self, details, workflow_manage): - self.context['result'] = details.get('result') - self.context['tool_params'] = details.get('tool_params') - self.context['mcp_tool'] = details.get('mcp_tool') + self.context.update({key: details.get(key) for key in ('result', 'text', 'structured', 'tool_params', 'mcp_tool', 'mcp_server', 'attempts', 'task_id')}) - def execute(self, mcp_servers, mcp_server, mcp_tool, mcp_tool_id, mcp_source, tool_params,**kwargs) -> NodeResult: - if mcp_source == 'referencing': - if not mcp_tool_id: - raise ValueError("MCP tool ID is required when mcp_source is 'referencing'.") - tool = QuerySet(Tool).filter(id=mcp_tool_id).first() - if not tool: - raise ValueError(f"Tool with ID {mcp_tool_id} not found.") - if not tool.is_active: - raise ValueError(f"Tool with ID {mcp_tool_id} is inactive.") - servers = json.loads(tool.code) - servers = self.handle_variables(servers) # 处理servers中的变量 - params = json.loads(json.dumps(tool_params)) - params = self.handle_variables(params) - else: - servers = json.loads(mcp_servers) - servers = self.handle_variables(servers) # 处理servers中的变量 - params = json.loads(json.dumps(tool_params)) - params = self.handle_variables(params) + def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, mcp_tool_id=None, mcp_source=None, + tool_form_field=None, timeout=60, retries=0, priority=5, **kwargs): + body, user = workflow_identity(self.workflow_manage) + params = self.handle_variables(deepcopy(tool_params), tool_form_field or []) + key = 'mcp:' + str(mcp_tool_id) if mcp_source == 'referencing' else 'mcp-custom' + result = mcp_runtime.invoke(body['workspace_id'], user, key, + {'servers': self.handle_variables(mcp_runtime.connections(mcp_servers)) if key == 'mcp-custom' else None, + 'server': mcp_server, 'tool': mcp_tool, 'arguments': params}, + parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'), + scope={**body, 'node_id': self.id, 'source': 'workflow-mcp'}, timeout=timeout, retries=retries, priority=priority) + return NodeResult({**result, 'tool_params': params}, {}) - async def call_tool(t, a): - client = MultiServerMCPClient(servers) - async with client.session(mcp_server) as s: - return await s.call_tool(t, a) - - res = asyncio.run(call_tool(mcp_tool, params)) - return NodeResult( - {'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {}) - - def handle_variables(self, tool_params): - # 处理参数中的变量 - for k, v in tool_params.items(): - if type(v) == str: - tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k]) - if type(v) == dict: - self.handle_variables(v) - if (type(v) == list) and (type(v[0]) == str): - tool_params[k] = self.get_reference_content(v) - return tool_params + def handle_variables(self, value, fields=None): + if isinstance(value, dict): + output = {} + for key, item in value.items(): + field = next((f for f in fields or [] if f.get('field') == key), {}) + if field.get('source') == 'custom' and field.get('input_type') == 'JsonInput' and isinstance(item, str): + try: output[key] = json.loads(self.workflow_manage.generate_prompt(item)) + except ValueError: raise ValidationError('工具参数必须为有效 JSON') from None + elif field.get('source') == 'custom': + output[key] = self.workflow_manage.generate_prompt(item) if isinstance(item, str) else item + elif field.get('source') == 'referencing': + if not isinstance(item, list) or len(item) < 2: raise ValidationError('请选择工具参数的上游变量') + output[key] = self.get_reference_content(item) + else: + output[key] = self.handle_variables(item, fields) + return output + if isinstance(value, str): return self.workflow_manage.generate_prompt(value) + if isinstance(value, list): + known = set(self.workflow_manage.flow.node_map) | {'global', 'chat', 'loop'} + if len(value) >= 2 and all(isinstance(v, str) for v in value) and value[0] in known: + return self.get_reference_content(value) + return [self.handle_variables(item, fields) for item in value] + return value def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field( - fields[0], - fields[1:]) + return self.workflow_manage.get_reference_field(fields[0], fields[1:]) def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'status': self.status, - 'err_message': self.err_message, - 'type': self.node.type, - 'mcp_tool': self.context.get('mcp_tool'), - 'tool_params': self.context.get('tool_params'), - 'result': self.context.get('result'), - } + return {'name': self.node.properties.get('stepName'), 'index': index, + 'run_time': self.context.get('run_time'), 'status': self.status, + 'err_message': self.err_message, 'type': self.node.type, + **{key: self.context.get(key) for key in ('mcp_tool', 'mcp_server', 'tool_params', 'result', 'text', 'structured', 'attempts', 'task_id')}} diff --git a/apps/application/flow/tools.py b/apps/application/flow/tools.py index 41348c6e5..8978b89e6 100644 --- a/apps/application/flow/tools.py +++ b/apps/application/flow/tools.py @@ -13,7 +13,7 @@ import threading from typing import Iterator from django.http import StreamingHttpResponse -from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk +from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk, AIMessage, SystemMessage from langchain_mcp_adapters.client import MultiServerMCPClient from langgraph.prebuilt import create_react_agent from application.flow.i_step_node import WorkFlowPostHandler @@ -267,17 +267,23 @@ def get_global_loop(): return _global_loop -async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True): - client = MultiServerMCPClient(json.loads(mcp_servers)) - tools = await client.get_tools() +async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None): + servers = json.loads(mcp_servers) + tools = await MultiServerMCPClient(servers).get_tools() if servers else [] + if engine_context: + from system_manage.mcp_agent import tools as engine_tools + tools += await engine_tools(engine_context) agent = create_react_agent(chat_model, tools) - response = agent.astream({"messages": message_list}, stream_mode='messages') + messages = message_list + if not any(isinstance(message, SystemMessage) for message in message_list): + messages = [SystemMessage(content='你是应用助手。根据用户请求选择已授权工具。需要外部数据或生成图谱时实际调用工具,不要编造工具结果。工具执行完成后依据结果回答,保存导图后返回打开链接,避免重复调用。'), *message_list] + response = agent.astream({"messages": messages}, stream_mode='messages') # 用于存储工具调用信息 tool_calls_info = {} async for chunk in response: - if isinstance(chunk[0], AIMessageChunk): + if isinstance(chunk[0], (AIMessageChunk, AIMessage)): tool_calls = chunk[0].additional_kwargs.get('tool_calls', []) for tool_call in tool_calls: tool_id = tool_call.get('id', '') @@ -287,6 +293,10 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_ 'name': tool_call.get('function', {}).get('name', ''), 'input': tool_call.get('function', {}).get('arguments', '') } + for tool_call in chunk[0].tool_calls: + if tool_call.get('id'): + tool_calls_info[tool_call['id']] = {'name': tool_call['name'], + 'input': json.dumps(tool_call.get('args', {}), ensure_ascii=False)} yield chunk[0] if mcp_output_enable and isinstance(chunk[0], ToolMessage): @@ -303,14 +313,30 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_ yield chunk[0] -def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True): +def collect_mcp_response(response): + """Use the same tool stream for the non-streaming chat API.""" + from langchain_core.messages import AIMessage + parts, reasoning, metadata = [], [], {} + for chunk in response: + if isinstance(chunk.content, str): + parts.append(chunk.content) + else: + parts.extend(item.get('text', '') for item in chunk.content if isinstance(item, dict)) + metadata.update(chunk.response_metadata) + value = chunk.additional_kwargs.get('reasoning_content') + if value: reasoning.append(value) + if reasoning: metadata['reasoning_content'] = ''.join(reasoning) + return AIMessage(content=''.join(parts), response_metadata=metadata) + + +def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None): """使用全局事件循环,不创建新实例""" result_queue = queue.Queue() loop = get_global_loop() # 使用共享循环 async def _run(): try: - async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable) + async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable, engine_context) async for chunk in async_gen: result_queue.put(('data', chunk)) except Exception as e: diff --git a/apps/application/serializers/application.py b/apps/application/serializers/application.py index 9b9c67424..becdd18d2 100644 --- a/apps/application/serializers/application.py +++ b/apps/application/serializers/application.py @@ -151,6 +151,8 @@ class ModelKnowledgeAssociation(serializers.Serializer): class ModelSettingSerializer(serializers.Serializer): + engine_tools = serializers.ListField(required=False, child=serializers.ChoiceField( + choices=['knowledge-search', 'mindmap-save']), max_length=2) prompt = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400, label=_("Prompt word")) system = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400, @@ -677,21 +679,8 @@ class ApplicationOperateSerializer(serializers.Serializer): if with_valid: self.is_valid(raise_exception=True) McpServersSerializer(data=instance).is_valid(raise_exception=True) - servers = json.loads(instance.get('mcp_servers')) - for server, config in servers.items(): - if config.get('transport') not in ['sse', 'streamable_http']: - raise AppApiException(500, _('Only support transport=sse or transport=streamable_http')) - tools = [] - for server in servers: - tools += [ - { - 'server': server, - 'name': tool.name, - 'description': tool.description, - 'args_schema': tool.args_schema, - } - for tool in asyncio.run(get_mcp_tools({server: servers[server]}))] - return tools + from system_manage.mcp_runtime import discover + return discover(instance.get('mcp_servers')) def delete(self, with_valid=True): if with_valid: diff --git a/apps/application/views/application_chat.py b/apps/application/views/application_chat.py index 66c85d827..97ce89a06 100644 --- a/apps/application/views/application_chat.py +++ b/apps/application/views/application_chat.py @@ -136,7 +136,7 @@ class OpenView(APIView): enforce(request, 'application', application_id) return result.success(OpenChatSerializers( data={'workspace_id': workspace_id, 'application_id': application_id, - 'chat_user_id': str(uuid.uuid7()), 'chat_user_type': ChatUserType.ANONYMOUS_USER, + 'chat_user_id': str(request.user.id), 'chat_user_type': ChatUserType.PLATFORM_USER.value, 'debug': True}).open()) diff --git a/apps/common/config/tokenizer_manage_config.py b/apps/common/config/tokenizer_manage_config.py index 9a3ae73f2..af80a7cd2 100644 --- a/apps/common/config/tokenizer_manage_config.py +++ b/apps/common/config/tokenizer_manage_config.py @@ -8,6 +8,9 @@ """ import os +import logging +import threading +from pathlib import Path class MKTokenizer: def __init__(self, tokenizer): @@ -17,14 +20,38 @@ class MKTokenizer: return self.tokenizer.encode(text).ids +class ByteTokenEstimate: + """Conservative token budget when an offline tokenizer is unavailable. + + Provider-reported usage remains authoritative. This only supplies local + estimates for prompt budgeting; it never downloads model files. + """ + def encode(self, text): + return list(text.encode('utf-8')) + + class TokenizerManage: tokenizer = None + _lock = threading.Lock() @staticmethod def get_tokenizer(): - from tokenizers import Tokenizer - # 创建Tokenizer - model_path = os.path.join("/opt/maxkb-app", "model", "tokenizer", "models--bert-base-cased") - with open(f"{model_path}/refs/main", encoding="utf-8") as f: snapshot = f.read() - TokenizerManage.tokenizer = Tokenizer.from_file(f"{model_path}/snapshots/{snapshot}/tokenizer.json") - return MKTokenizer(TokenizerManage.tokenizer) + if TokenizerManage.tokenizer is not None: + return TokenizerManage.tokenizer + with TokenizerManage._lock: + if TokenizerManage.tokenizer is not None: + return TokenizerManage.tokenizer + from tokenizers import Tokenizer + model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or + '/opt/maxkb-app/model/tokenizer/models--bert-base-cased') + try: + path = model_path if model_path.is_file() else model_path / 'tokenizer.json' + if not path.is_file(): + snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip() + path = model_path / 'snapshots' / snapshot / 'tokenizer.json' + TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path))) + except Exception: + # A missing/corrupt tokenizer must not prevent remote-model chat. + logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准') + TokenizerManage.tokenizer = ByteTokenEstimate() + return TokenizerManage.tokenizer diff --git a/apps/common/handle/impl/text/xlsx_split_handle.py b/apps/common/handle/impl/text/xlsx_split_handle.py index fd467363d..3215bdcf9 100644 --- a/apps/common/handle/impl/text/xlsx_split_handle.py +++ b/apps/common/handle/impl/text/xlsx_split_handle.py @@ -54,7 +54,7 @@ class XlsxSplitHandle(BaseSplitHandle): try: if type(limit) is str: limit = int(limit) - workbook = openpyxl.load_workbook(io.BytesIO(buffer)) + workbook = openpyxl.load_workbook(io.BytesIO(buffer), data_only=True) try: image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer)) save_image([item for item in image_dict.values()]) diff --git a/apps/knowledge/embedding_params.py b/apps/knowledge/embedding_params.py index 6f1eeb329..3f01b657e 100644 --- a/apps/knowledge/embedding_params.py +++ b/apps/knowledge/embedding_params.py @@ -37,6 +37,7 @@ def validate_params(params): def knowledge_params(knowledge): params = validate_params((knowledge.meta or {}).get('embedding_params') or {}) + params.setdefault('normalize', True) return {key: value for key, value in params.items() if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}} @@ -48,7 +49,7 @@ class ConfiguredEmbedding(Embeddings): self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10 def _vector(self, vector): - if not self.params.get('normalize'): + if not self.params.get('normalize', True): return vector norm = math.sqrt(sum(float(v) ** 2 for v in vector)) return [float(v) / norm for v in vector] if norm else vector diff --git a/apps/knowledge/serializers/document.py b/apps/knowledge/serializers/document.py index 528920349..9e4afaa31 100644 --- a/apps/knowledge/serializers/document.py +++ b/apps/knowledge/serializers/document.py @@ -1057,7 +1057,7 @@ class DocumentSerializers(serializers.Serializer): knowledge = QuerySet(Knowledge).filter(id=self.data.get('knowledge_id')).first() segmentation = (knowledge.meta or {}).get('segmentation') if knowledge is not None else None segmentation = segmentation or {} - default_limit = segmentation.get('chunk_size') or 4096 + default_limit = segmentation.get('chunk_size') or 500 default_overlap = segmentation.get('chunk_overlap') or 0 limit = instance.get("limit", None) or default_limit overlap = instance.get("overlap", None) @@ -1740,7 +1740,7 @@ class DocumentSerializers(serializers.Serializer): data={'workspace_id': self.data.get('workspace_id'), 'knowledge_id': self.data.get('knowledge_id')} ) split_serializer.is_valid(instance={'file': [file]}, raise_exception=True) - parsed_documents = split_serializer.file_to_paragraph(file, None, None, 4096) + parsed_documents = split_serializer.file_to_paragraph(file, None, None, 500) paragraphs = [] for parsed_document in parsed_documents: paragraphs.extend(parsed_document.get('content', [])) diff --git a/apps/knowledge/serializers/knowledge.py b/apps/knowledge/serializers/knowledge.py index 62f294a5f..c69dfc9d6 100644 --- a/apps/knowledge/serializers/knowledge.py +++ b/apps/knowledge/serializers/knowledge.py @@ -346,9 +346,12 @@ class KnowledgeSerializer(serializers.Serializer): ) ) ), with_search_one=True) + from knowledge.embedding_guard import embedding_change_reason return { **knowledge_dict, 'meta': json.loads(knowledge_dict.get('meta', '{}')), + 'embedding_change_block_reason': embedding_change_reason( + QuerySet(Knowledge).get(id=self.data.get('knowledge_id'))), 'application_id_list': list(filter( lambda application_id: all_application_list.__contains__(application_id), [ @@ -371,6 +374,23 @@ class KnowledgeSerializer(serializers.Serializer): ).exclude(id=knowledge.id).exists(): raise AppApiException(500, _('Knowledge base name duplicate!')) KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge) + from knowledge.embedding_params import knowledge_params, validate_params + from knowledge.embedding_guard import embedding_change_reason + from types import SimpleNamespace + current_params = knowledge_params(knowledge) + try: + params = validate_params(instance['embedding_params']) if 'embedding_params' in instance else None + except ValueError as exc: + raise AppApiException(500, str(exc)) + new_params = knowledge_params(SimpleNamespace(meta={'embedding_params': params})) if params is not None else current_params + model_changed = ('embedding_model_id' in instance and + str(instance['embedding_model_id'] or '') != str(knowledge.embedding_model_id or '')) + vector_params_changed = ({k: v for k, v in current_params.items() if k != 'batch_size'} != + {k: v for k, v in new_params.items() if k != 'batch_size'}) + if model_changed or vector_params_changed: + reason = embedding_change_reason(knowledge) + if reason: + raise AppApiException(500, reason) if 'embedding_model_id' in instance: knowledge.embedding_model_id = instance.get('embedding_model_id') if "name" in instance: @@ -386,11 +406,6 @@ class KnowledgeSerializer(serializers.Serializer): incoming_meta[managed_key] = knowledge.meta[managed_key] knowledge.meta = incoming_meta if 'embedding_params' in instance: - from knowledge.embedding_params import validate_params - try: - params = validate_params(instance['embedding_params']) - except ValueError as exc: - raise AppApiException(500, str(exc)) knowledge.meta = {**(knowledge.meta or {}), 'embedding_params': params} if 'folder_id' in instance: knowledge.folder_id = instance.get('folder_id') diff --git a/apps/knowledge/serializers/knowledge_ext.py b/apps/knowledge/serializers/knowledge_ext.py index 495dd73eb..08eda23d1 100644 --- a/apps/knowledge/serializers/knowledge_ext.py +++ b/apps/knowledge/serializers/knowledge_ext.py @@ -166,7 +166,7 @@ class KnowledgeSegmentationSerializer(serializers.Serializer): 保存后新建文档上传时默认使用该规则,上传时可覆盖。 """ - DEFAULTS = {"chunk_size": 4096, "chunk_overlap": 0} + DEFAULTS = {"chunk_size": 500, "chunk_overlap": 0} @staticmethod def one(workspace_id: str, knowledge_id: str): diff --git a/apps/knowledge/test_embedding_params.py b/apps/knowledge/test_embedding_params.py index b2af2d073..9ed884234 100644 --- a/apps/knowledge/test_embedding_params.py +++ b/apps/knowledge/test_embedding_params.py @@ -54,19 +54,29 @@ class EmbeddingParamsTests(unittest.TestCase): params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'} self.assertEqual(validate_params(params), params) active = knowledge_params(SimpleNamespace(meta={'embedding_params': params})) - self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:'}) + self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:', 'normalize': True}) base = RecordingModel() model = ConfiguredEmbedding(base, active) model.embed_query('问题') self.assertEqual(base.calls, [('query', '检索:问题')]) - def test_legacy_and_empty_params_keep_original_behavior(self): - self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {}) + def test_missing_normalization_defaults_to_unit_vectors(self): + self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {'normalize': True}) + self.assertEqual(knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': None}})), + {'normalize': True}) self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {}) base = RecordingModel() - self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [3., 4.]) + self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [.6, .8]) + self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents(['x']), [[.6, .8]]) self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), []) + def test_explicit_false_keeps_original_vectors(self): + params = knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': False}})) + self.assertEqual(params, {'normalize': False}) + model = ConfiguredEmbedding(RecordingModel(), params) + self.assertEqual(model.embed_query('x'), [3., 4.]) + self.assertEqual(model.embed_documents(['x']), [[3., 4.]]) + def test_zero_vector_is_safe(self): base = RecordingModel() base.embed_query = lambda _: [0., 0.] diff --git a/apps/knowledge/test_knowledge_ext.py b/apps/knowledge/test_knowledge_ext.py index 99e8a504d..3cbf64537 100644 --- a/apps/knowledge/test_knowledge_ext.py +++ b/apps/knowledge/test_knowledge_ext.py @@ -152,6 +152,11 @@ class MetadataTests(TestCase): def test_upload_inherits_defaults_and_accepts_explicit_override(self): from knowledge.serializers.document import DocumentSerializers + self.assertEqual(Rules.one('default', str(self.k.id))['chunk_size'], 500) + serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'}) + with patch.object(serializer,'file_to_paragraph',return_value=[]) as split: + serializer.parse({'file':[SimpleUploadedFile('default.txt',b'test')]}) + self.assertEqual(split.call_args.args[-2:], (500,0)) Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20}) serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'}) with patch.object(serializer,'file_to_paragraph',return_value=[]) as split: diff --git a/apps/models_provider/base_model_provider.py b/apps/models_provider/base_model_provider.py index b77df219c..6977d6436 100644 --- a/apps/models_provider/base_model_provider.py +++ b/apps/models_provider/base_model_provider.py @@ -97,8 +97,11 @@ class MaxKBBaseModel(ABC): @staticmethod def filter_optional_params(model_kwargs): + from system_manage.summary_plan import LOCAL_PARAMETERS optional_params = {} for key, value in model_kwargs.items(): + if key in LOCAL_PARAMETERS and key != 'timeout': + continue if key not in ['model_id', 'use_local', 'streaming', 'show_ref_label', 'stream']: if key == 'extra_body' and isinstance(value, dict): optional_params = {**optional_params, **value} diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py index 44b078c7d..9e6b9db86 100644 --- a/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/model/llm.py @@ -14,6 +14,9 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI): @staticmethod def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs) + # Client controls belong to the SDK, not the provider's JSON body. + client_params = {key: optional_params.pop(key) for key in + ('temperature', 'top_p', 'max_tokens', 'timeout', 'max_retries') if key in optional_params} # if 'qwen-omni-turbo' in model_name or 'qwq' in model_name: # optional_params['streaming'] = True return BaiLianChatModel( @@ -21,5 +24,7 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI): openai_api_base=model_credential.get('api_base'), openai_api_key=model_credential.get('api_key'), streaming=True, + stream_usage=True, + **client_params, extra_body=optional_params ) diff --git a/apps/models_provider/impl/ollama_model_provider/model/llm.py b/apps/models_provider/impl/ollama_model_provider/model/llm.py index 6a2db1aa9..140453d60 100644 --- a/apps/models_provider/impl/ollama_model_provider/model/llm.py +++ b/apps/models_provider/impl/ollama_model_provider/model/llm.py @@ -36,6 +36,8 @@ class OllamaChatModel(MaxKBBaseModel, ChatOllama): api_base = model_credential.get('api_base', '') base_url = get_base_url(api_base) optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs) + from models_provider.context_policy import model_context + optional_params['num_ctx'] = model_context(model_kwargs) return OllamaChatModel(model=model_name, base_url=base_url, model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None, diff --git a/apps/models_provider/local_imports.py b/apps/models_provider/local_imports.py index 0fa4f1204..c9ae03e92 100644 --- a/apps/models_provider/local_imports.py +++ b/apps/models_provider/local_imports.py @@ -1,11 +1,13 @@ """Model file assets, independent of runtime/model connection records. Never executes model code.""" import json +import hashlib import os import shutil import stat import tempfile import uuid import zipfile +from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path, PurePosixPath @@ -18,6 +20,42 @@ class ImportError(ValueError): pass +def model_fingerprint(directory): + """Hash relative paths and complete file contents, independent of the archive name.""" + root = Path(directory) + digest = hashlib.sha256() + for file in sorted((p for p in root.rglob('*') if p.is_file()), key=lambda p: p.relative_to(root).as_posix()): + name = file.relative_to(root).as_posix().encode('utf-8') + digest.update(len(name).to_bytes(8, 'big')) + digest.update(name) + digest.update(file.stat().st_size.to_bytes(8, 'big')) + with file.open('rb') as source: + for chunk in iter(lambda: source.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +@contextmanager +def import_lock(root): + """Serialize duplicate checks and record creation across Web processes.""" + with (root / '.import.lock').open('a+b') as lock: + if os.name == 'nt': + import msvcrt + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + if os.name == 'nt': + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + def inspect_model(directory): root = Path(directory) if not root.is_dir() or root.is_symlink(): @@ -98,14 +136,33 @@ class ModelImportStore: self.root = Path(root) self.root.mkdir(parents=True, exist_ok=True) - def list(self): + def _records(self): records = [] for path in self.root.glob('*/import.json'): + if path.parent.name.startswith('.import-'): + continue try: records.append(json.loads(path.read_text(encoding='utf-8'))) except (ValueError, OSError): continue - return sorted(records, key=lambda r: r['created_at'], reverse=True)[:200] + return sorted(records, key=lambda r: r.get('created_at', ''), reverse=True) + + def list(self): + records, paths, fingerprints = [], set(), set() + for record in self._records(): + path = record.get('path') + canonical = os.path.normcase(os.path.realpath(path)) if path else None + fingerprint = record.get('fingerprint') if record.get('source') == 'UPLOAD' else None + if (canonical and canonical in paths) or (fingerprint and fingerprint in fingerprints): + continue + if canonical: + paths.add(canonical) + if fingerprint: + fingerprints.add(fingerprint) + records.append(record) + if len(records) == 200: + break + return records def create(self, owner_id, path=None, upload=None): if (path is None) == (upload is None): @@ -163,15 +220,37 @@ class ModelImportStore: model_dir = Path(path).resolve(strict=True) info = inspect_model(model_dir) final_path, source_type, display_name = model_dir, 'PATH', model_dir.name - record = { - 'id': identity, 'name': display_name, 'path': str(final_path), - 'source': source_type, 'owner_id': str(owner_id), - 'created_at': datetime.now(timezone.utc).isoformat(), - 'status': 'IMPORTED', **info, - } - (staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8') - staging.rename(self.root / identity) - return record + with import_lock(self.root): + existing = self._records() + if source_type == 'PATH': + canonical = os.path.normcase(os.path.realpath(model_dir)) + for record in existing: + if record.get('path') and os.path.normcase(os.path.realpath(record['path'])) == canonical: + return {**record, 'already_imported': True} + fingerprint = model_fingerprint(model_dir) + for record in existing: + if (record.get('size_bytes') != info['size_bytes'] or + record.get('file_count') != info['file_count']): + continue + previous_path = record.get('path') + if not previous_path or not Path(previous_path).is_dir(): + continue + try: + inspect_model(previous_path) + previous = model_fingerprint(previous_path) + except (OSError, ImportError): + continue + if previous == fingerprint: + return {**record, 'already_imported': True} + record = { + 'id': identity, 'name': display_name, 'path': str(final_path), + 'source': source_type, 'owner_id': str(owner_id), + 'created_at': datetime.now(timezone.utc).isoformat(), + 'status': 'IMPORTED', 'fingerprint': fingerprint, **info, + } + (staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8') + staging.rename(self.root / identity) + return record except (FileNotFoundError, PermissionError) as exc: raise ImportError('目录不存在或无访问权限') from exc finally: diff --git a/apps/models_provider/test_local_imports.py b/apps/models_provider/test_local_imports.py index add79d88d..774e5dd60 100644 --- a/apps/models_provider/test_local_imports.py +++ b/apps/models_provider/test_local_imports.py @@ -105,5 +105,54 @@ class ImportTests(unittest.TestCase): for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}): with self.assertRaises(ImportError):self.store.create('owner',**kwargs) + def test_repeated_path_and_historical_records_are_deduplicated(self): + first = self.store.create('owner', path=str(self.model)) + repeated = self.store.create('owner', path=str(self.model / '..' / 'model')) + self.assertEqual(repeated['id'], first['id']) + self.assertTrue(repeated['already_imported']) + legacy = self.store.root / 'legacy' + legacy.mkdir() + (legacy / 'import.json').write_text(json.dumps({**first, 'id': 'legacy', 'created_at': '2000'})) + self.assertEqual([row['id'] for row in self.store.list()], [first['id']]) + + def test_zip_content_deduplicates_and_distinct_content_is_retained(self): + files = [('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')] + first = self.store.create('owner', upload=Upload(files)) + second = self.store.create('owner', upload=Upload(list(reversed(files)))) + self.assertEqual(first['id'], second['id']) + self.assertTrue(second['already_imported']) + different = self.store.create('owner', upload=Upload([ + ('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights2')])) + self.assertNotEqual(first['id'], different['id']) + self.assertEqual(len(self.store.list()), 2) + + def test_different_directory_and_zip_with_same_files_reuse_record(self): + first = self.store.create('owner', path=str(self.model)) + copy = self.root / 'copy' + import shutil + shutil.copytree(self.model, copy) + second = self.store.create('owner', path=str(copy)) + self.assertEqual(first['id'], second['id']) + zipped = Upload([('repo/config.json', (copy / 'config.json').read_bytes()), + ('repo/model.safetensors', (copy / 'model.safetensors').read_bytes())]) + self.assertEqual(self.store.create('owner', upload=zipped)['id'], first['id']) + + def test_concurrent_imports_create_one_record(self): + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(lambda _: self.store.create('owner', path=str(self.model)), range(4))) + self.assertEqual(len({row['id'] for row in results}), 1) + self.assertEqual(len(self.store._records()), 1) + + def test_invalid_legacy_directory_does_not_block_import(self): + first = self.store.create('owner', path=str(self.model)) + config = self.model / 'config.json' + config.write_text('x' * len(config.read_text())) + replacement = self.root / 'replacement' + replacement.mkdir() + (replacement / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'})) + (replacement / 'model.safetensors').write_bytes(b'test fixture, not runtime weights') + self.assertNotEqual(self.store.create('owner', path=str(replacement))['id'], first['id']) + if __name__ == '__main__':unittest.main() diff --git a/apps/models_provider/tools.py b/apps/models_provider/tools.py index e178bb1dc..64e53f979 100644 --- a/apps/models_provider/tools.py +++ b/apps/models_provider/tools.py @@ -131,11 +131,15 @@ def get_model_default_params(model): return value return value - return { + params = { p.get('field'): convert_to_int(p.get('default_value')) for p in model.model_params_form if p.get('default_value') is not None } + if model.model_type == 'LLM': + from models_provider.context_policy import model_context + params['context_tokens'] = model_context(params) + return params def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs): @@ -147,4 +151,8 @@ def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs): """ model = get_model_by_id(model_id, workspace_id) s = get_model_default_params(model) - return ModelManage.get_model(model_id, lambda _id: get_model(model, **{**s, **kwargs})) + params = {**s, **kwargs} + if model.model_type == 'LLM' and any(kwargs.get(key) is not None for key in ('context_tokens', 'num_ctx')): + from models_provider.context_policy import model_context + params['context_tokens'] = model_context(kwargs) + return ModelManage.get_model(model_id, lambda _id: get_model(model, **params)) diff --git a/apps/system_manage/ai_hub.py b/apps/system_manage/ai_hub.py index 2fca5c8a9..98c09bf67 100644 --- a/apps/system_manage/ai_hub.py +++ b/apps/system_manage/ai_hub.py @@ -144,19 +144,23 @@ def serialize(obj, detail=False, user=None): elif isinstance(obj, AIFlow): data.update(revision=obj.revision, nodes=obj.nodes) elif isinstance(obj, AIService): + from system_manage.ai_hub_service_api import get_config data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users, requests_per_minute=obj.requests_per_minute, - key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run') + key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run', api_config=get_config(obj.id)) elif isinstance(obj, AITask): from system_manage.ai_hub_progress import task_progress data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt, started_at=obj.started_at, finished_at=obj.finished_at) if detail: data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj)) + if any(n.get('type') == 'organize' for n in obj.snapshot.get('nodes', [])): + data['summary_input'] = {key: obj.input_data[key] for key in ('markdown', 'heading_structure') if key in obj.input_data} elif isinstance(obj, AIArtifact): data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None) if detail: - data.update(original=obj.original, markdown=obj.markdown, history=obj.history) + from system_manage.heading_structure import artifact_state, version_history, summary_state + data.update(original=obj.original, markdown=obj.markdown, history=version_history(obj), heading_structure=artifact_state(obj), summary_state=summary_state(obj)) return data @@ -251,22 +255,24 @@ def validate_nodes(workspace, user, nodes): raise ValidationError('解析服务已停用') clean['resource_id'] = str(parser.id) if kind == 'organize': + if node.get('operation') not in (None, 'heading_repair'): + raise ValidationError('整理操作无效') + if node.get('operation'): + clean['operation'] = node['operation'] clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id) clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000) - temperature = node.get('temperature') - if temperature is not None: - if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: - raise ValidationError('温度无效') - clean['temperature'] = temperature - # Absent parameters fall back to the per-model settings in model management. - for field, low, high in ( - ('context_tokens', 4096, 262144), ('output_tokens', 256, 32768), - ('safety_tokens', 256, 8192), ('timeout', 30, 600)): - value = node.get(field) - if value is not None: - clean[field] = integer(value, low, high, '模型参数') - if clean.get('output_tokens', 4096) + clean.get('safety_tokens', 1024) >= clean.get('context_tokens', 32768): - raise ValidationError('输出与预留长度不能超过模型上下文') + from system_manage.summary_plan import validate_options + from system_manage.ai_hub_organize import resolve_settings + from models_provider.tools import get_model_default_params + from system_manage.mcp_business import summary_model_defaults, summary_supported_parameters + try: + clean.update(validate_options(node)) + model = model_for(workspace, user, clean['model_id']) + resolve_settings(summary_model_defaults(model, get_model_default_params(model)), clean) + if clean.get('top_p') is not None and 'top_p' not in summary_supported_parameters(model): + raise ValueError('该模型未声明支持 top_p 参数') + except ValueError as exc: + raise ValidationError(str(exc)) from None result.append(clean) return result @@ -286,10 +292,14 @@ def service_create(workspace, user, data): require_admin(user) flow = one('flows', data.get('flow_id'), workspace, user, lock=True) validate_nodes(workspace, user, flow.nodes) - return AIService.objects.create(workspace_id=workspace, owner_id=user.id, + obj = AIService.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name'), '名称', limit=128), flow=flow, allowed_users=users(data.get('allowed_users', [])), requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率')) + if 'api_config' in data: + from system_manage.ai_hub_service_api import save_config + save_config(obj.id, data['api_config']) + return obj def make_snapshot(workspace, user, nodes): @@ -324,6 +334,10 @@ def submit(workspace, user, data, upload=None, service=None, request=None): else: nodes = one('flows', data['flow_id'], workspace, user, lock=True).nodes if data.get('flow_id') else data.get('nodes') snapshot = make_snapshot(workspace, user, nodes) + from system_manage.mcp_registry import pipeline_resources, resolve + required = pipeline_resources(snapshot) + for key in required: resolve(workspace, user, key) + priority = integer(data.get('priority', 5), 0, 9, '优先级') from system_manage.access_control import enforce_task enforce_task(request, service.id if service else None, snapshot) markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT) @@ -338,33 +352,98 @@ def submit(workspace, user, data, upload=None, service=None, request=None): from system_manage.ai_hub_assets import asset_lock, normalize with asset_lock(): markdown = text(normalize(markdown, workspace, user), '文档内容', required=False) + heading_input = {} + if data.get('heading_structure') is not None: + from system_manage.heading_structure import analyze, validate_manual + if not isinstance(data['heading_structure'], dict): + raise ValidationError('层级数据无效') + try: + _, structure = analyze(markdown, previous=data['heading_structure'], repair=False) + structure = validate_manual(structure, data['heading_structure']) + except ValueError as exc: + raise ValidationError(str(exc)) from None + heading_input = {'heading_structure': structure, 'original': text(data.get('original', markdown), '原稿', required=False)} + if data.get('_heading_source') is not None: + heading_input['heading_source'] = data['_heading_source'] + # Plan without invoking a model. Reserve only useful internal parallel calls. + from system_manage.mcp_registry import settings_row + from system_manage.mcp_business import summary_model_defaults, summary_counter + from system_manage.summary_plan import token_counter + from system_manage.ai_hub_organize import plan_document, resolve_settings + from models_provider.tools import get_model_default_params + from system_manage.ai_hub_assets import protect_images + scheduler_settings = settings_row().meta + capacity = scheduler_settings.get('capacity', 4) + worker_capacity = max(1, capacity - min(scheduler_settings.get('interactive_reserve', 1), max(0, capacity - 1))) + resource_slots = {} + for node in snapshot: + if node['type'] != 'organize' or node.get('operation') == 'heading_repair': + continue + key = 'model:' + node['model_id'] + slots = min(node.get('concurrency', 1) if node.get('parallel', True) else 1, + resolve(workspace, user, key)[0]['capacity'], worker_capacity) + if snapshot[0]['type'] != 'parse': + model = model_for(workspace, user, node['model_id']) + defaults = summary_model_defaults(model, get_model_default_params(model)) + try: + options = {**node, 'max_input_tokens': min(int(defaults.get('max_input_tokens', 2000000)), node.get('max_input_tokens', 2000000))} + if 'input_tokens' not in options and defaults.get('input_tokens'): + options['input_tokens'] = int(defaults['input_tokens']) + settings = resolve_settings(defaults, node) + count, _ = summary_counter(model, defaults, markdown, workspace, user.id, settings, options) + if node.get('structure_mode', 'preserve') in ('preserve', 'reorganize'): + from system_manage.summary_sections import build_plan + planned = build_plan(protect_images(markdown)[0], node['prompt'], settings, options, count) + slots = min(slots, max(1, len(planned['batches']))) + else: + _, _, _, _, chunks = plan_document(protect_images(markdown)[0], node['prompt'], settings, options, count) + slots = min(slots, len(chunks)) + except ValueError as exc: + raise ValidationError(str(exc)) from None + resource_slots[key] = slots return AITask.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service, - snapshot={'nodes': snapshot, 'flow_id': str(service.flow_id) if service else data.get('flow_id'), + snapshot={'nodes': snapshot, 'resources': required, 'resource_slots': resource_slots, 'priority': priority, 'flow_id': str(service.flow_id) if service else data.get('flow_id'), 'flow_revision': service.flow.revision if service else None}, - input_data={'markdown': markdown}, file_name=upload.name[:255] if upload else '', + input_data={'markdown': markdown, **heading_input}, file_name=upload.name[:255] if upload else '', file_data=upload.read() if upload else None) +@transaction.atomic +def artifact_rename(obj, data): + if set(data) != {'action', 'name', 'revision'}: + raise ValidationError('重命名参数无效') + if isinstance(data['revision'], bool) or not isinstance(data['revision'], int) or data['revision'] != obj.revision: + raise ValidationError('内容已更新,请重新打开') + obj.name = text(data['name'], '名称', limit=128) + obj.save(update_fields=['name', 'updated_at']) + return obj + + @transaction.atomic def artifact_save(workspace, user, data, obj=None): + from system_manage.heading_structure import analyze, artifact_state, store_artifact_state, store_summary_state, summary_state, validate_manual + previous_structure = artifact_state(obj) if obj else {} if obj: obj = AIArtifact.objects.select_for_update().get(pk=obj.pk) + previous_structure = artifact_state(obj) if data.get('revision') != obj.revision: raise ValidationError('内容已更新,请重新打开') - history = list(obj.history) - if len(history) >= 30: - raise ValidationError('修订已达30次,请另存为新图谱') - history.append({'revision': obj.revision, 'markdown': obj.markdown, 'layout': obj.layout, - 'operator': str(user.id), 'updated_at': obj.updated_at.isoformat()}) - obj.history = history obj.revision += 1 else: + from users.models import User + from system_manage.artifact_settings import limit + User.objects.select_for_update().get(pk=user.id) + maximum = limit() + if AIArtifact.objects.filter(workspace_id=workspace, owner_id=user.id).count() >= maximum: + raise ValidationError(f'已达到导图保存上限({maximum}条),请删除后再保存') task = one('tasks', data['task_id'], workspace, user, lock=True) if data.get('task_id') else None if task and task.status != 'SUCCESS': raise ValidationError('任务尚未成功') obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task, original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT)) + if task: + previous_structure = (task.result.get('source') or {}).get('heading_structure', {}) obj.name = text(data.get('name'), '名称', limit=128) obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT) obj.layout = data.get('layout', 'tree') @@ -374,8 +453,34 @@ def artifact_save(workspace, user, data, obj=None): with asset_lock(): obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容') obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False) + if 'summary_state' in data: + state = data['summary_state'] + if not isinstance(state, dict) or set(state) - {'source_markdown', 'smart_markdown', 'mode'}: + raise ValidationError('智能总结内容无效') + if state.get('mode', 'source') not in ('source', 'smart'): + raise ValidationError('智能总结状态无效') + clean_state = {'mode': state.get('mode', 'source')} + for key in ('source_markdown', 'smart_markdown'): + clean_state[key] = text(normalize(text(state.get(key, ''), '总结内容', required=False, limit=MAX_INLINE_TEXT), workspace, user), '总结内容', required=False, limit=MAX_INLINE_TEXT) + store_summary_state(obj, clean_state) + elif summary_state(obj): + state = dict(summary_state(obj)) + state.pop('_type', None) + state['smart_markdown' if state.get('mode') == 'smart' else 'source_markdown'] = obj.markdown + store_summary_state(obj, state) for entry in obj.history: - entry['markdown'] = normalize(entry.get('markdown', ''), workspace, user) + if 'markdown' in entry: + entry['markdown'] = normalize(entry['markdown'], workspace, user) + incoming = data.get('heading_structure') + if incoming is not None or previous_structure: + try: + _, structure = analyze(obj.markdown, previous=previous_structure or incoming, repair=False) + if incoming is not None: + structure = validate_manual(structure, incoming) + _, structure = analyze(obj.markdown, previous=structure, repair=False) + except ValueError as exc: + raise ValidationError(str(exc)) from None + store_artifact_state(obj, structure) obj.save() return obj @@ -407,6 +512,9 @@ def delete_record(kind, pk, workspace, user): from system_manage.models.platform_access import PlatformGrant PlatformGrant.objects.filter(workspace_id=workspace, resource_type=resource_type, target=str(obj.id)).delete() + if kind == 'services': + from system_manage.ai_hub_service_api import save_config + save_config(obj.id) obj.delete() @@ -419,9 +527,10 @@ def parse_file(parser, name, content, options=None, progress_callback=None): return parse(parser, name, content, opts, headers, progress_callback=progress_callback) -def markdown_tree(markdown): +def markdown_tree(markdown, excluded_lines=None): """Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。""" import re + excluded_lines = set(excluded_lines or []) root = {'id': 'root', 'text': '文档', 'children': []} heading_stack = [(0, root)] list_stack = [] @@ -459,6 +568,8 @@ def markdown_tree(markdown): list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$') for index, line in enumerate(markdown.splitlines()): + if excluded_lines and index in excluded_lines: + continue fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line) if fence_match: token = fence_match.group(1) diff --git a/apps/system_manage/ai_hub_assets.py b/apps/system_manage/ai_hub_assets.py index 873e2d8e1..45b63b631 100644 --- a/apps/system_manage/ai_hub_assets.py +++ b/apps/system_manage/ai_hub_assets.py @@ -172,7 +172,7 @@ def protect_images(markdown): return pattern.sub(replace, markdown), images -def restore_images(markdown, images): +def restore_images(markdown, images, append_missing=True): # Summarizers may omit or repeat placeholders. The backend owns the image # manifest, so preserve known positions and append omitted originals once. known = {str(index + 1): image for index, image in enumerate(images)} @@ -185,7 +185,7 @@ def restore_images(markdown, images): return known[index] markdown = re.sub(r'AIHUBIMAGE[\s_*`-]*(\d+)[\s_*`-]*TOKEN', replace, markdown, flags=re.I) missing = [image for index, image in known.items() if index not in restored] - if missing: + if missing and append_missing: markdown = markdown.rstrip() + '\n\n## 原文图片\n\n' + '\n\n'.join(missing) return markdown diff --git a/apps/system_manage/ai_hub_organize.py b/apps/system_manage/ai_hub_organize.py index d3db06b77..5245e81f0 100644 --- a/apps/system_manage/ai_hub_organize.py +++ b/apps/system_manage/ai_hub_organize.py @@ -1,6 +1,41 @@ """Bounded hierarchical organization; source and completed chunks survive failures.""" import re import time +import hashlib +import json +import threading +from models_provider.context_policy import DEFAULT_CONTEXT_TOKENS, summary_defaults +from concurrent.futures import ThreadPoolExecutor, as_completed +from system_manage.summary_plan import LOCAL_PARAMETERS, prompt_rules, output_estimate, source_language, language_rule, language_matches, boundaries_match + +ORGANIZER_PARAMETERS = LOCAL_PARAMETERS + + +def resolve_settings(defaults, params): + """Workflow overrides win; model-management fields configure the organizer locally.""" + defaults = summary_defaults(defaults) + def number(name, value, low, high): + if isinstance(value, str) and value.isdigit(): + value = int(value) + if type(value) is not int or not low <= value <= high: + raise ValueError(f'总结参数 {name} 无效,请填写 {low} 至 {high} 的整数') + return value + context = number('context_tokens', params.get('context_tokens', defaults['context_tokens']), 1024, 2000000) + if context > int(defaults['context_tokens']): + raise ValueError('上下文超过模型配置上限,请先在模型管理中配置能力参数') + output = number('output_tokens', params.get('max_tokens', params.get('output_tokens', + defaults['output_tokens'])), 1, 524288) + safety = number('safety_tokens', params.get('safety_tokens', defaults['safety_tokens']), 0, 200000) + if output + safety >= context: + raise ValueError('总结输出与预留长度不能超过上下文长度') + for key, value in [('context_tokens', context), ('max_input_tokens', params.get('max_input_tokens')), + ('max_output_tokens', max(output, params.get('max_output_tokens') or 0))]: + limit = defaults.get(key) + if limit is not None and value is not None and int(value) > int(limit): + raise ValueError(f'总结参数超过模型配置的 {key} 上限') + if params.get('max_output_tokens') is not None and output > params['max_output_tokens']: + raise ValueError('本次输出长度超过模型最大输出长度') + return {'context_tokens': context, 'output_tokens': output, 'safety_tokens': safety} def token_bound(text): @@ -8,25 +43,31 @@ def token_bound(text): return len(text.encode('utf-8')) -def split_markdown(text, budget): +def split_markdown(text, budget, count=None): + count = count or token_bound if budget < 256: raise ValueError('模型上下文预算不足,请调大上下文或减少输出长度') chunks, current = [], '' - blocks = re.split(r'(?m)(?=^#{1,6} )', text) + # Do not mistake headings inside fenced examples for document sections. + protected = list(re.finditer(r'(?ms)^```[^\n]*\n.*?^```[^\n]*(?:\n|$)|^~~~[^\n]*\n.*?^~~~[^\n]*(?:\n|$)', text)) + boundaries = [m.start() for m in re.finditer(r'(?m)^#{1,6} ', text) + if not any(span.start() <= m.start() < span.end() for span in protected)] + boundaries = sorted(set([0, *boundaries, len(text)])) + blocks = [text[a:b] for a, b in zip(boundaries, boundaries[1:])] for block in blocks: while block: - if token_bound(current + block) <= budget: + if count(current + block) <= budget: current += block break if current: chunks.append(current); current = '' - if token_bound(block) <= budget: + if count(block) <= budget: current = block break - lo, hi = 1, min(len(block), budget) + lo, hi = 1, len(block) while lo < hi: mid = (lo + hi + 1) // 2 - if token_bound(block[:mid]) <= budget: + if count(block[:mid]) <= budget: lo = mid else: hi = mid - 1 @@ -34,6 +75,13 @@ def split_markdown(text, budget): newline = block.rfind('\n', 0, end) if newline > end // 2: end = newline + 1 + # Keep a table, fenced example, or consecutive list together when + # the whole block fits; oversized blocks may still be continued. + spans = re.finditer(r'(?ms)^```[^\n]*\n.*?^```[^\n]*(?:\n|$)|^~~~[^\n]*\n.*?^~~~[^\n]*(?:\n|$)|(?:^\|[^\n]*\n?)+|(?:^[ \t]*(?:[-*+] |\d+[.)] )[^\n]*\n?)+', block) + for span in spans: + if span.start() < end < span.end() and count(span[0]) <= budget: + end = span.start() if span.start() else span.end() + break # Never split an image placeholder between requests. for match in re.finditer(r'AIHUBIMAGE\s*\d+\s*TOKEN', block[:end + 64]): if match.start() < end < match.end(): @@ -47,65 +95,301 @@ def split_markdown(text, budget): return chunks -def organize(markdown, prompt, invoke, *, context_tokens=32768, output_tokens=4096, - safety_tokens=1024, progress=None, checkpoint=None): - budget = context_tokens - output_tokens - safety_tokens - token_bound(prompt) - 512 - chunks = split_markdown(markdown, budget) +def plan_document(markdown, prompt, settings, options, count=token_bound): + context_tokens, output_tokens, safety_tokens = (settings[key] for key in ('context_tokens', 'output_tokens', 'safety_tokens')) + instruction = prompt_rules(prompt, options, source_language(markdown, prompt)) + # Includes bounded batch context, retry guidance, and message framing. + input_limit = min(context_tokens - output_tokens - safety_tokens, + options.get('max_input_tokens') or context_tokens) + budget = input_limit - count(instruction) - 2048 + if budget < 256: + raise ValueError('模型上下文预算不足,请减少提示词或输出预留') + ratio = max(.05, output_estimate(markdown, count, options) / max(1, count(markdown))) + # Output risk can require splitting even when input fits. Never constrain + # every map result to a fraction of the final merge's output allowance. + output_budget = max(256, int(output_tokens * .8 / ratio)) + chunk_budget = min(budget, output_budget) + chunks = split_markdown(markdown, chunk_budget, count) if len(chunks) > 128: - raise ValueError('文档分块过多,请调大模型上下文或拆分文档') - calls = 0 + raise ValueError('文档分块过多,请调大模型预算或拆分文档') + if len(chunks) > 1 and not options.get('auto_split', True): + raise ValueError('文档超过输入或输出预算,请开启自动拆分') + return instruction, input_limit, budget, chunk_budget, chunks - def call(content, instruction): - nonlocal calls - calls += 1 - if calls > 160: - raise ValueError('整理批次过多,请拆分文档') - if token_bound(content) + token_bound(instruction) > context_tokens - output_tokens - safety_tokens: + +def organize(markdown, prompt, invoke, *, context_tokens=DEFAULT_CONTEXT_TOKENS, output_tokens=None, + safety_tokens=None, progress=None, checkpoint=None, metrics=None, + options=None, count=None, count_method='conservative_bytes', resume=None, + state_checkpoint=None, fingerprint_context=None): + options = options or {} + count = count or token_bound + inherited = summary_defaults({'context_tokens': context_tokens}) + output_tokens = inherited['output_tokens'] if output_tokens is None else output_tokens + safety_tokens = inherited['safety_tokens'] if safety_tokens is None else safety_tokens + settings = {'context_tokens': context_tokens, 'output_tokens': output_tokens, 'safety_tokens': safety_tokens} + instruction, input_limit, budget, chunk_budget, chunks = plan_document(markdown, prompt, settings, options, count) + merge = len(chunks) > 1 and options.get('structure_mode', 'preserve') == 'reorganize' + concurrency = min(len(chunks), options.get('concurrency', 1)) if options.get('parallel', True) else 1 + stats = metrics if metrics is not None else {} + stats.update(context_tokens=context_tokens, output_tokens=output_tokens, safety_tokens=safety_tokens, + input_bytes=token_bound(markdown), input_tokens_estimate=count(markdown), token_count_method=count_method, + input_budget=budget, chunk_budget=chunk_budget, map_chunks=len(chunks), concurrency=concurrency, + needs_merge=merge, expected_calls=len(chunks) + int(merge), model_calls=0, + compression_retries=0, language_retries=0, structure_retries=0, fact_retries=0, language_splits=0, split_retries=0, reused_chunks=0, + estimated_output_tokens=output_estimate(markdown, count, options)) + fingerprint = hashlib.sha256(json.dumps([markdown, instruction, context_tokens, output_tokens, + safety_tokens, chunk_budget, options, fingerprint_context, 'summary-v4'], + ensure_ascii=False, sort_keys=True).encode()).hexdigest() + previous = (resume or {}).get('outputs', {}) if (resume or {}).get('fingerprint') == fingerprint else {} + saved = {key: value for key, value in previous.items() if isinstance(value, str) and value.strip()} + lock = threading.RLock() + last_saved = None + + def persist(): + nonlocal last_saved + with lock: + if checkpoint and saved != last_saved: + checkpoint(list(saved.values())) + last_saved = dict(saved) + if state_checkpoint: + state_checkpoint({'fingerprint': fingerprint, 'outputs': dict(saved), 'metrics': dict(stats)}) + + def report(stage, percent): + if progress: + progress({'stage': stage, 'percent': percent, 'plan': dict(stats)}) + + def call(content, guidance): + if count(content) + count(guidance) + 512 > input_limit: raise ValueError('模型输入超过上下文预算') - for attempt in range(3): + for attempt in range(options.get('retries', 2) + 1): + with lock: + stats['model_calls'] += 1 + if stats['model_calls'] > 160: + raise ValueError('总结调用超过限制,已保留完成分块') try: - response = invoke(instruction, content) + response = invoke(guidance, content) break except Exception as exc: - if attempt == 2: + if attempt == options.get('retries', 2): raise ValueError('模型调用失败,已保留原稿和完成的分块,请稍后重试') from exc time.sleep(attempt + 1) - metadata = getattr(response, 'response_metadata', {}) - if isinstance(metadata, dict) and metadata.get('finish_reason') in ('length', 'max_tokens'): - raise ValueError('模型输出被截断,请到模型管理调大该模型的输出上限,或减少整理内容') - result = getattr(response, 'content', response) - if isinstance(result, list): - result = ''.join(item.get('text', '') for item in result if isinstance(item, dict)) - if not isinstance(result, str) or not result.strip(): + metadata = getattr(response, 'response_metadata', {}) or {} + truncated = (metadata.get('finish_reason') or metadata.get('done_reason')) in ('length', 'max_tokens', 'max_output_tokens') + value = getattr(response, 'content', response) + if isinstance(value, list): + value = ''.join(item.get('text', '') for item in value if isinstance(item, dict)) + if not truncated and (not isinstance(value, str) or not value.strip()): raise ValueError('模型未返回有效整理内容') - return result.strip() + value = value.strip() if isinstance(value, str) else '' + if value.startswith('```'): + lines = value.splitlines() + closing = next((i for i, line in enumerate(lines[1:], 1) if re.fullmatch(r'```\s*', line)), len(lines)) + value = '\n'.join(lines[1:closing]).strip() + elif '```' not in content: + closing = re.search(r'(?m)^```\s*$', value) + if closing: + value = value[:closing.start()].strip() + usage = getattr(response, 'usage_metadata', {}) or metadata.get('token_usage', {}) or {} + actual_output = usage.get('output_tokens', usage.get('completion_tokens', metadata.get('eval_count'))) + return value, truncated, actual_output - outputs = [] - chunk_prompt = prompt + '\n这是文档的一个分块,保留事实及标题关系,精简重复内容。' - for index, chunk in enumerate(chunks, 1): - if progress: - progress({'stage': f'智能整理分块 {index}/{len(chunks)}', 'percent': (index - 1) * 70 // len(chunks)}) - outputs.append(call(chunk, prompt if len(chunks) == 1 else chunk_prompt)) - if checkpoint: - checkpoint(list(outputs)) - if len(outputs) == 1: - return outputs[0] - merge_prompt = prompt + '\n合并以下局部大纲,统一标题层级,压缩重复内容;保留事实、数字和图片占位符。' - previous_size = None - for round_index in range(4): - joined = '\n\n'.join(outputs) - groups = split_markdown(joined, budget) - if previous_size is not None and token_bound(joined) >= previous_size and len(groups) > 1: - raise ValueError('合并结果未收敛,已保留完成的分块,请缩小文档或调整提示词') - previous_size = token_bound(joined) - merged = [] - for index, group in enumerate(groups, 1): - if progress: - progress({'stage': f'合并大纲 {round_index + 1}:{index}/{len(groups)}', 'percent': 70 + round_index * 6}) - merged.append(call(group, merge_prompt)) - if checkpoint: - checkpoint(merged) - if len(merged) == 1: - return merged[0] - outputs = merged - raise ValueError('合并批次超过限制,已保留完成的分块') + def summarize(content, key, guidance, depth=0): + with lock: + if key in saved: + stats['reused_chunks'] += 1 + return saved[key] + original = content + language = source_language(original, prompt) + language_failed = False + outline_source = re.sub(r'(?ms)^```[^\n]*\n.*?^```[^\n]*(?:\n|$)|^~~~[^\n]*\n.*?^~~~[^\n]*(?:\n|$)', '', original) + required_headings = re.findall(r'(?m)^(#{1,6}) (.+)$', outline_source) if options.get('structure_mode') == 'preserve' else [] + outline = '\n'.join(hashes + ' ' + title for hashes, title in required_headings) + # An exceptionally large TOC would defeat the body budget; it is already + # present in the source. Include only bounded, exact outline guidance. + outline_hint = ('\nREQUIRED OUTPUT OUTLINE. Copy these headings EXACTLY and add summary bullets beneath each:\n' + outline + if outline and count(outline) <= 400 else '') + structure_failed = False + facts_failed = False + for attempt in range(2): + hint = guidance + f'\nOutput at most {output_tokens} tokens. Summarize concisely, do not repeat the full source.' + if attempt: + hint += '\nPrevious result failed validation. Return a concise, complete summary in the required language.' + with lock: + stats['language_retries' if language_failed else 'structure_retries' if structure_failed else 'fact_retries' if facts_failed else 'compression_retries'] += 1 + hint += '\n' + language_rule(language) + hint += outline_hint + if language_failed or structure_failed: + hint += '\nVALIDATION FAILED. Keep ALL original section headings in order. Fill EACH section using its source language.' + if facts_failed: + hint += '\nNUMERIC CONDITION VALIDATION FAILED. Preserve all explicit comparison operators and conditions EXACTLY. Above/exceeds means >, not >= or reaches. Do not weaken or reverse a threshold.' + value, truncated, actual_output = call(content, hint) + source_sections = list(re.finditer(r'(?m)^(#{1,6}) (.+)$', outline_source)) if required_headings else [] + target_sections = list(re.finditer(r'(?m)^(#{1,6}) (.+)$', value)) + bodies = [outline_source[src.end():source_sections[j+1].start() if j+1 < len(source_sections) else len(outline_source)].strip() + for j, src in enumerate(source_sections)] + # A heading-only ancestor chain has one unambiguous body owner. + # Restore that skeleton in code instead of paying for a relabel call. + if (source_sections and sum(bool(body) for body in bodies) == 1 and bodies[-1] and target_sections and + all(not value[dst.end():target_sections[j+1].start()].strip() for j, dst in enumerate(target_sections[:-1]))): + last_body = value[target_sections[-1].end():].strip() + if (len(target_sections) <= len(source_sections) and + language_matches(last_body, source_language(bodies[-1], prompt)) and + set(re.findall(r'\d+(?:\.\d+)?', bodies[-1])) <= set(re.findall(r'\d+(?:\.\d+)?', last_body))): + value = '\n'.join(src[1] + ' ' + src[2] for src in source_sections) + '\n' + last_body + target_sections = list(re.finditer(r'(?m)^(#{1,6}) (.+)$', value)) + aligned = bool(source_sections) and len(source_sections) == len(target_sections) + section_language_failed = False + if aligned: + # Headings are owned by the document. Restore harmless translated + # or reformatted labels only when section order, body languages, + # and numeric anchors agree; do not ask the model to relabel them. + for j, (src, dst) in enumerate(zip(source_sections, target_sections)): + body = outline_source[src.end():source_sections[j+1].start() if j+1 < len(source_sections) else len(outline_source)].strip() + summary = value[dst.end():target_sections[j+1].start() if j+1 < len(target_sections) else len(value)].strip() + anchors = set(re.findall(r'\d+(?:\.\d+)?', body)) + if body and not language_matches(summary, source_language(body, prompt)): + section_language_failed = True + aligned = False + if not anchors <= set(re.findall(r'\d+(?:\.\d+)?', summary)): + aligned = False + if aligned: + for src, dst in reversed(list(zip(source_sections, target_sections))): + value = value[:dst.start()] + src[1] + ' ' + src[2] + value[dst.end():] + # With no model tokenizer use the conservative estimator, explicitly + # recorded in the plan. Provider finish_reason remains authoritative. + language_failed = section_language_failed or not language_matches(value, language) + facts_failed = not boundaries_match(original, value) + actual_headings = re.findall(r'(?m)^#{1,6} (.+)$', value) + remaining = iter(actual_headings) + heading_key = lambda title: re.sub(r'[*_`]', '', title).strip() + structure_failed = any(not any(heading_key(candidate) == heading_key(title) for candidate in remaining) for _, title in required_headings) + # Some small models translate mixed-language documents despite valid + # instructions. Retry only language-local sections, with no global + # rewrite. Models that succeed initially still use a single call. + body_languages = {source_language(body, prompt) for body in bodies if body} + if (attempt == 0 and depth < 2 and language == 'mixed' and (language_failed or structure_failed) + and len(body_languages) > 1 and body_languages <= {'english', 'chinese'}): + with lock: + stats['language_splits'] += 1 + stats['expected_calls'] += sum(bool(body) for body in bodies) + parts = [] + for j, (section, body) in enumerate(zip(source_sections, bodies)): + heading = section[1] + ' ' + section[2] + if not body: + parts.append(heading) + continue + section_language = source_language(body, prompt) + parts.append(summarize(heading + '\n' + body, key + '/language' + str(j), + prompt_rules(prompt, options, section_language) + + f'\nLanguage-local recovery section {j+1}/{len(source_sections)}. Only summarize this section.', depth+1)) + value = '\n\n'.join(parts) + with lock: + saved[key] = value + return value + if not truncated and not language_failed and not structure_failed and not facts_failed and (actual_output if isinstance(actual_output, int) else count(value)) <= output_tokens: + # Restore original levels by ordered occurrence, including duplicate titles. + pending_headings = list(required_headings) + def restore_level(match): + if pending_headings and heading_key(match[2]) == heading_key(pending_headings[0][1]): + hashes, title = pending_headings.pop(0) + return hashes + ' ' + title + return match[0] + value = re.sub(r'(?m)^(#{1,6}) (.+)$', restore_level, value) + with lock: + saved[key] = value + return value + if language_failed or structure_failed or facts_failed: + content = original + elif not truncated and count(value) <= budget: + content = value + if language_failed: + raise ValueError('总结未保持原文语言,已保留完成分块;请调整提示词或更换模型后重试') + if structure_failed: + raise ValueError('总结未保留原文标题,已保留完成分块;请调整提示词或更换模型后重试') + if facts_failed: + raise ValueError('总结改变了数值边界条件,已保留原稿和完成分块;请调整提示词或更换模型后重试') + if depth >= 3 or count(original) <= 512: + raise ValueError('模型总结截断或长度校验未通过,已保留完成的分块,请提高输出上限或减少细节') + with lock: + stats['split_retries'] += 1 + parts = split_markdown(original, max(256, (count(original) + 1) // 2), count) + values = [summarize(part, key + '/' + str(i), guidance + + f'\n本批继续拆分为 {len(parts)} 段,当前第 {i+1} 段,只总结当前正文。', depth + 1) + for i, part in enumerate(parts)] + value = '\n\n'.join(values) + with lock: + saved[key] = value + return value + + report(f'预计 {len(chunks)} 批,并发 {concurrency},' + ('完成后汇总' if merge else '完成后组装'), 0) + outputs = [None] * len(chunks) + offset = 0 + jobs = [] + for i, chunk in enumerate(chunks): + prefix = markdown[:offset] + path = [] + for match in re.finditer(r'(?m)^(#{1,6}) (.+)$', prefix): + level, title = len(match[1]), match[2] + path = [(n, t) for n, t in path if n < level] + [(level, title)] + headings = re.findall(r'(?m)^#{1,6} (.+)$', chunk) + opening_heading = re.match(r'^(#{1,6}) ', chunk) + if opening_heading: + path = [(level, title) for level, title in path if level < len(opening_heading[1])] + metadata = ('\nStage: full document.' if len(chunks) == 1 else + f'\nCONTEXT ONLY, NEVER INCLUDE IN OUTPUT: batch {i+1}/{len(chunks)}; ' + f'ancestor path: {" > ".join(t for _, t in path)[-80:] or "document start"}; ' + f'headings: {" > ".join(headings[:3])[:80] or "continued body"}; ' + f'source character range {offset+1}-{offset+len(chunk)} of {len(markdown)}; ' + f'previous/next batch: {i or "none"}/{i+2 if i+1 < len(chunks) else "none"}. ' + 'Summarize ONLY the current source, never infer other batches.' + + (f'\nPrevious source excerpt for context only, do not summarize it again: {prefix[-80:]}' + if prefix and not opening_heading else '')) + jobs.append((chunk, f'part{i}', instruction + metadata)) + offset += len(chunk) + error = None + if concurrency == 1: + for i, job in enumerate(jobs): + try: + outputs[i] = summarize(*job) + finally: + persist() + report(f'总结分块 {i+1}/{len(chunks)}', (i+1)*85//len(chunks)) + else: + with ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = {pool.submit(summarize, *job): i for i, job in enumerate(jobs)} + done = 0 + for future in as_completed(futures): + try: + outputs[futures[future]] = future.result() + except Exception as exc: + error = error or exc + for pending in futures: + pending.cancel() + done += 1 + persist() + report(f'总结分块 {done}/{len(chunks)}', done*85//len(chunks)) + if error: + raise error + if merge: + for round_index in range(4): + joined = '\n\n'.join(outputs) + groups = split_markdown(joined, chunk_budget, count) + report(f'合并大纲 {round_index+1},共 {len(groups)} 批', 88 + round_index*2) + merged = [] + for i, group in enumerate(groups): + try: + merged.append(summarize(group, f'merge{round_index}/{i}', instruction + + f'\n全局汇总阶段,当前第 {i+1}/{len(groups)} 组章节摘要。统一主题、去重,保持原文语言及不同条件。')) + finally: + persist() + if len(merged) == 1: + outputs = merged + break + if count('\n\n'.join(merged)) >= count(joined): + raise ValueError('合并结果未收敛,已保留完成的分块') + outputs = merged + else: + raise ValueError('合并批次超过限制,已保留完成的分块') + report('总结完成', 100) + persist() + return '\n\n'.join(outputs) diff --git a/apps/system_manage/ai_hub_progress.py b/apps/system_manage/ai_hub_progress.py index dd030224e..d12c31ec8 100644 --- a/apps/system_manage/ai_hub_progress.py +++ b/apps/system_manage/ai_hub_progress.py @@ -34,4 +34,5 @@ def task_progress(task): if percent is not None: percent = min(99, int((complete_steps * 100 + percent) / total_steps)) return {'stage': detail.get('stage') or LABELS.get(current.get('type'), '处理中'), - 'percent': percent, 'elapsed_seconds': elapsed} + 'percent': percent, 'elapsed_seconds': elapsed, + **({'plan': detail['plan']} if isinstance(detail.get('plan'), dict) else {})} diff --git a/apps/system_manage/ai_hub_worker.py b/apps/system_manage/ai_hub_worker.py index ce1f5cf7c..8ebad058d 100644 --- a/apps/system_manage/ai_hub_worker.py +++ b/apps/system_manage/ai_hub_worker.py @@ -6,18 +6,21 @@ from system_manage import ai_hub as hub def claim(): - with transaction.atomic(): - task = AITask.objects.select_for_update(skip_locked=True).filter(status='PENDING').order_by('created_at').first() - if not task: - return None - task.status = 'RUNNING' - task.started_at = timezone.now() - task.save(update_fields=['status', 'started_at', 'updated_at']) - return task + from system_manage.mcp_scheduler import admit + return admit('worker') def execute(task): from users.models import User + if task.snapshot.get('engine'): + try: + from system_manage.mcp_workflows import execute as execute_workflow + return execute_workflow(task) + except Exception: + from system_manage.mcp_scheduler import finish + finish(task, 'FAILED', error='流程执行失败,请检查组件、授权及流程版本') + task.refresh_from_db() + return task steps = [] try: user = User.objects.get(pk=task.owner_id) @@ -31,17 +34,31 @@ def execute(task): hub.require_use(user, service) if not service.enabled: raise ValueError('服务已停用') + from system_manage.mcp_registry import pipeline_resources, resolve + for key in pipeline_resources(task.snapshot['nodes']): resolve(task.workspace_id, user, key, check=False) markdown = task.input_data.get('markdown', '') - original = markdown - source = {} + original = task.input_data.get('original', markdown) + source = task.input_data.get('heading_source', {}) + if task.input_data.get('heading_structure'): + source = {**source, 'heading_structure': task.input_data['heading_structure']} tree = {} - task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree} + previous_state = task.result.get('summary_state') + previous_parsed = task.result.get('parse_checkpoint') + task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree, + **({'summary_state': previous_state} if previous_state else {}), + **({'parse_checkpoint': previous_parsed} if previous_parsed else {})} task.save(update_fields=['result', 'updated_at']) for node in task.snapshot['nodes']: step = {'type': node['type'], 'status': 'RUNNING'} steps.append(step) task.steps = steps task.save(update_fields=['steps', 'updated_at']) + if node['type'] == 'parse' and previous_parsed and previous_parsed.get('node') == node: + markdown, original, source = (previous_parsed[key] for key in ('markdown', 'original', 'source')) + step.update(status='SUCCESS', progress={'stage': '复用已完成解析', 'percent': 100}) + task.result.update(markdown=markdown, original=original, source=source) + task.save(update_fields=['steps', 'result', 'updated_at']) + continue if node['type'] == 'parse': parser = None if node.get('processor') != 'builtin': @@ -54,63 +71,65 @@ def execute(task): AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update( steps=steps, updated_at=timezone.now()) report_progress({'stage': '上传文档到解析服务', 'percent': None}) - if parser is None: - from system_manage.parsers.native import parse - report_progress({'stage': '使用内置处理器解析文档', 'percent': None}) - parsed = parse(task.file_name, bytes(task.file_data)) - else: - parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) - from system_manage.ai_hub_assets import asset_lock, normalize - with asset_lock(): - markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果') - source = parsed['source'] - # Source JSON can contain image data too; preserve structure without inline payloads. - import json - source = json.loads(normalize(json.dumps(source), task.workspace_id, user)) - task.result = {'original': original, 'markdown': markdown, 'source': source} - task.save(update_fields=['result', 'updated_at']) + from system_manage import mcp_runtime + key = 'file-reader' if parser is None else 'parser:' + str(parser.id) + parsed = mcp_runtime.invoke(task.workspace_id, user, key, {'options': node.get('options', {})}, + parent=task.id, progress=report_progress, scope={'source': 'mindmap'}) + markdown = parsed['text'] + original = parsed.get('original', markdown) + source = parsed['structured'] + task.refresh_from_db(fields=['result']) + task.result.update(original=original, markdown=markdown, source=source, + parse_checkpoint={'node': node, 'markdown': markdown, 'original': original, 'source': source}) + task.save(update_fields=['result', 'updated_at']) elif node['type'] == 'organize': - from models_provider.tools import get_model, get_model_default_params - from langchain_core.messages import HumanMessage, SystemMessage - model = hub.model_for(task.workspace_id, user, node['model_id']) - if model.update_time.isoformat() != node['resource_version']: - raise ValueError('模型配置已变更,请重新提交') - # Bound input; never silently truncate the original document. - from system_manage.ai_hub_assets import protect_images, restore_images - model_markdown, image_tokens = protect_images(markdown) - try: - # Per-model settings from model management win; task nodes override only when set. - defaults = get_model_default_params(model) - temperature = node.get('temperature') - if temperature is None: - temperature = defaults.get('temperature', 0.2) - max_tokens = node.get('output_tokens') - if max_tokens is None: - max_tokens = defaults.get('max_tokens', 4096) - instance = get_model(model, **{**defaults, 'temperature': temperature, - 'max_tokens': max_tokens, 'timeout': node.get('timeout', 180)}) - prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。' - except Exception as exc: - from rest_framework.exceptions import ValidationError - raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc - from system_manage.ai_hub_organize import organize - def organize_progress(progress): - step['progress'] = progress - AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now()) - def checkpoint(outputs): - task.result['organize_chunks'] = outputs - task.save(update_fields=['result', 'updated_at']) - organized = organize(model_markdown, prompt, - lambda instruction, content: instance.invoke([SystemMessage(content=instruction), HumanMessage(content=content)]), - context_tokens=node.get('context_tokens', 32768), output_tokens=max_tokens, - safety_tokens=node.get('safety_tokens', 1024), progress=organize_progress, checkpoint=checkpoint) - markdown = restore_images(hub.text(organized, '整理结果'), image_tokens) + if node.get('operation') == 'heading_repair': + from system_manage.heading_service import invoke_model + def heading_progress(progress): + step['progress'] = progress + AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now()) + organized = invoke_model(task.workspace_id, user, node, markdown, source, task.id, heading_progress) + markdown = organized['text'] + source = {**source, **organized['structured']} + else: + from system_manage import mcp_runtime + model = hub.model_for(task.workspace_id, user, node['model_id']) + if model.update_time.isoformat() != node['resource_version']: + raise ValueError('模型配置已变更,请重新提交') + def organize_progress(progress): + step['progress'] = progress + AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now()) + def checkpoint(outputs): + task.refresh_from_db(fields=['result']) + task.result['summary_state' if isinstance(outputs, dict) else 'organize_chunks'] = outputs + if isinstance(outputs, dict) and outputs.get('metrics'): + step.setdefault('progress', {})['plan'] = outputs['metrics'] + task.save(update_fields=['result', 'steps', 'updated_at']) + params = {'text': markdown, 'instruction': node['prompt'], + 'operation': node.get('operation'), + 'heading_structure': source.get('heading_structure', {}), 'heading_source': source, + 'resume': task.result.get('summary_state'), + **{key: node[key] for key in ( + 'temperature', 'top_p', 'context_tokens', 'max_input_tokens', 'max_output_tokens', + 'safety_tokens', 'retries', 'detail', 'structure_mode', 'parallel', 'auto_split', + 'max_depth', 'max_children', 'node_max_chars', 'concurrency', 'input_tokens') if key in node}} + if node.get('output_tokens') is not None: params['max_tokens'] = node['output_tokens'] + organized = mcp_runtime.invoke(task.workspace_id, user, 'model:' + str(model.id), params, + parent=task.id, timeout=node.get('timeout', 180), progress=organize_progress, + checkpoint=checkpoint, scope={'source': 'mindmap'}) + markdown = organized['text'] + if isinstance(organized.get('structured'), dict): + source = {**source, **organized['structured']} else: - tree = hub.markdown_tree(markdown) + from system_manage import mcp_runtime + tree = mcp_runtime.invoke(task.workspace_id, user, 'mindmap-engine', {'text': markdown, + 'heading_structure': source.get('heading_structure')}, + parent=task.id, scope={'source': 'mindmap'})['structured'] step['status'] = 'SUCCESS' task.steps = steps # Persist intermediate original output so a later model failure does not lose OCR work. - task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree} + task.refresh_from_db(fields=['result']) + task.result.update(original=original, markdown=markdown, source=source, tree=tree) task.save(update_fields=['steps', 'result', 'updated_at']) task.status = 'SUCCESS' except Exception as exc: diff --git a/apps/system_manage/management/commands/ai_hub_worker.py b/apps/system_manage/management/commands/ai_hub_worker.py index 9af28f978..33a236de7 100644 --- a/apps/system_manage/management/commands/ai_hub_worker.py +++ b/apps/system_manage/management/commands/ai_hub_worker.py @@ -1,37 +1,65 @@ +import signal +import threading import time -from datetime import timedelta +from concurrent.futures import ThreadPoolExecutor from django.core.management.base import BaseCommand from django.db import close_old_connections from django.utils import timezone from system_manage.models.ai_hub import AITask from system_manage.ai_hub_worker import claim, execute +from system_manage import mcp_scheduler class Command(BaseCommand): - help = 'Run the isolated AI hub task consumer (does not run migrations).' + help = 'Run the isolated component/hub consumer; never consumes Celery queues or migrates.' def add_arguments(self, parser): parser.add_argument('--once', action='store_true') - parser.add_argument('--recover', action='store_true', help='Mark running tasks older than 30 minutes failed') + parser.add_argument('--recover', action='store_true', help='Mark tasks without heartbeat for 30 minutes failed') + parser.add_argument('--concurrency', type=int, default=1) def handle(self, *args, **options): - if options['recover']: - AITask.objects.filter(status='RUNNING', updated_at__lt=timezone.now() - timedelta(minutes=30)).update( - status='FAILED', error='执行中断,请重试', finished_at=timezone.now()) + concurrency = options.get('concurrency', 1) + if not 1 <= concurrency <= 8: + raise ValueError('concurrency must be between 1 and 8') + if options.get('recover'): + mcp_scheduler.recover() + stop = threading.Event() + if threading.current_thread() is threading.main_thread(): + signal.signal(signal.SIGTERM, lambda *_: stop.set()) next_cleanup = time.monotonic() + 86400 - while True: + def run(task): close_old_connections() - if time.monotonic() >= next_cleanup: - from system_manage.ai_hub_assets import cleanup - try: - cleanup(apply=True) - except Exception: - self.stderr.write('图片缓存清理失败,将于下次重试') - next_cleanup = time.monotonic() + 86400 - task = claim() - if task: - execute(task) - if options['once']: - return - if not task: - time.sleep(2) + try: + return execute(task) + except Exception: + mcp_scheduler.finish(task, 'FAILED', error='执行器异常,请检查后重试') + finally: + close_old_connections() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + running = {} + while not stop.is_set() or running: + close_old_connections() + mcp_scheduler.heartbeat() + for future, task in list(running.items()): + if future.done(): + future.result() + del running[future] + else: + AITask.objects.filter(pk=task.id, attempt=task.attempt, status='RUNNING').update(updated_at=timezone.now()) + if not stop.is_set(): + while len(running) < concurrency: + task = claim() + if not task: break + if stop.is_set(): + AITask.objects.filter(pk=task.id, attempt=task.attempt, status='RUNNING').update( + status='PENDING', started_at=None, updated_at=timezone.now()) + break + running[pool.submit(run, task)] = task + if options.get('once'): stop.set() + if time.monotonic() >= next_cleanup and not running: + from system_manage.ai_hub_assets import cleanup + try: cleanup(apply=True) + except Exception: self.stderr.write('图片缓存清理失败,将于下次重试') + next_cleanup = time.monotonic() + 86400 + if running or not stop.is_set(): time.sleep(1) diff --git a/apps/system_manage/parsers/native.py b/apps/system_manage/parsers/native.py index 7058d6b85..aa5f64367 100644 --- a/apps/system_manage/parsers/native.py +++ b/apps/system_manage/parsers/native.py @@ -15,6 +15,18 @@ def parse(name, content): from system_manage.ai_hub import text from . import extension suffix = extension(name) + if suffix in ('md', 'txt'): + # Preserve headings and whitespace even when a document has no body paragraph. + try: + markdown = content.decode('utf-8-sig') + except UnicodeDecodeError: + from charset_normalizer import from_bytes + decoded = from_bytes(content).best() + if decoded is None: + raise ValidationError('无法识别文本编码,请使用 UTF-8') + markdown = str(decoded) + return {'markdown': text(markdown, '解析结果'), 'images': {}, + 'source': {'engine': 'maxkb-native', 'format': suffix}} if suffix == 'doc': command = shutil.which('libreoffice') or shutil.which('soffice') if command is None: @@ -60,12 +72,31 @@ def parse(name, content): mime = mimetypes.guess_type(file.file_name)[0] or 'image/jpeg' images['./oss/file/' + str(file.id)] = f'data:{mime};base64,' + base64.b64encode(raw).decode() upload = SimpleUploadedFile(Path(name.replace('\\', '/')).name, content) - result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0) - sections = result if isinstance(result, list) else [result] - markdown = '\n\n'.join('\n\n'.join( - ((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', '')) - for item in section.get('content', [])) for section in sections if isinstance(section, dict)) + evidence = {} + if suffix == 'docx': + from docx import Document + from .native_outline import word_markdown + # Use the converter's Markdown before knowledge-base chunking flattens headings. + markdown, evidence = word_markdown(Document(io.BytesIO(content)), handler, capture) + elif suffix in ('html', 'htm'): + from html2text import html2text + from common.handle.impl.text.html_split_handle import get_encoding + markdown = html2text(content.decode(get_encoding(content))) + from .native_outline import html_evidence + evidence = html_evidence(content) + elif suffix == 'pdf': + from .native_outline import pdf_markdown + markdown, evidence = pdf_markdown(content) + if markdown is None: + result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0) + markdown = '\n\n'.join(item.get('content', '') for item in result.get('content', [])) + else: + result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0) + sections = result if isinstance(result, list) else [result] + markdown = '\n\n'.join('\n\n'.join( + ((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', '')) + for item in section.get('content', [])) for section in sections if isinstance(section, dict)) for path, uri in images.items(): markdown = markdown.replace(path, uri) return {'markdown': text(markdown, '解析结果'), 'images': {}, - 'source': {'engine': 'maxkb-native', 'format': suffix}} + 'source': {'engine': 'maxkb-native', 'format': suffix, **evidence}} diff --git a/apps/system_manage/test_ai_hub.py b/apps/system_manage/test_ai_hub.py index 3d8be5770..96071f36f 100644 --- a/apps/system_manage/test_ai_hub.py +++ b/apps/system_manage/test_ai_hub.py @@ -11,6 +11,83 @@ from system_manage.ai_hub_worker import claim, execute from system_manage.models.ai_hub import AIFlow, AIParserResource, AITask, AIArtifact, AIService +class NativeParserTests(TestCase): + def test_word_preserves_heading_only_sections_tables_and_images(self): + import io + from docx import Document + from PIL import Image + from system_manage.parsers.native import parse + + document = Document() + document.add_heading('Chapter42', 1) + document.add_heading('Section42', 2) + document.add_heading('Detail42', 3) + document.add_paragraph('Body42') + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).text = 'Name' + table.cell(0, 1).text = 'Value' + table.cell(1, 0).text = 'Count' + table.cell(1, 1).text = '42' + image = io.BytesIO() + Image.new('RGB', (2, 2), 'white').save(image, format='PNG') + image.seek(0) + document.add_picture(image) + buffer = io.BytesIO() + document.save(buffer) + + with patch('common.handle.impl.text.doc_split_handle.DocSplitHandle.handle', + side_effect=AssertionError('Knowledge chunking must not run')): + markdown = parse('headings.docx', buffer.getvalue())['markdown'] + self.assertEqual([line for line in markdown.splitlines() if line.startswith('#')], + ['# Chapter42', '## Section42', '### Detail42']) + self.assertIn('Body42', markdown) + self.assertIn('| Count | 42 |', markdown) + self.assertIn('data:image/png;base64,', markdown) + self.assertNotIn('./oss/file/', markdown) + tree = hub.markdown_tree(markdown) + self.assertEqual(tree['children'][0]['children'][0]['children'][0]['text'], 'Detail42') + + def test_html_and_htm_preserve_headings_lists_and_links_without_chunking(self): + from system_manage.parsers.native import parse + content = (b'

Chapter42

Section42

' + b'

Detail42

Body42

' + b'

Link42

') + for suffix in ('html', 'htm'): + with self.subTest(suffix=suffix), patch( + 'common.handle.impl.text.html_split_handle.HTMLSplitHandle.handle', + side_effect=AssertionError('Knowledge chunking must not run')): + markdown = parse('headings.' + suffix, content)['markdown'] + self.assertEqual([line for line in markdown.splitlines() if line.startswith('#')], + ['# Chapter42', '## Section42', '### Detail42']) + self.assertIn('Child42', markdown) + self.assertIn('[Link42](https://example.org)', markdown) + + def test_xlsx_uses_cached_formula_values_and_does_not_evaluate_formulas(self): + import io + import zipfile + from openpyxl import Workbook + from system_manage.parsers.native import parse + + workbook = Workbook() + workbook.active.append(['Cached', 'Uncached', 'Literal']) + workbook.active.append(['=6*7', '=2+3', 17]) + buffer = io.BytesIO() + workbook.save(buffer) + cached = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as source, zipfile.ZipFile(cached, 'w') as target: + for item in source.infolist(): + data = source.read(item.filename) + if item.filename == 'xl/worksheets/sheet1.xml': + self.assertIn(b'6*7', data) + data = data.replace(b'6*7', b'6*742') + target.writestr(item, data) + markdown = parse('cached.xlsx', cached.getvalue())['markdown'] + self.assertIn('| 42 | | 17 |', markdown) + self.assertNotIn('=6*7', markdown) + self.assertNotIn('=2+3', markdown) + + class HubTests(TestCase): def setUp(self): import tempfile diff --git a/apps/system_manage/test_ai_hub_organize.py b/apps/system_manage/test_ai_hub_organize.py index 3da33f69c..2656e5ad7 100644 --- a/apps/system_manage/test_ai_hub_organize.py +++ b/apps/system_manage/test_ai_hub_organize.py @@ -1,10 +1,59 @@ import unittest from types import SimpleNamespace from unittest.mock import patch -from system_manage.ai_hub_organize import organize, split_markdown, token_bound +from system_manage.ai_hub_organize import organize, split_markdown, token_bound, resolve_settings class OrganizeTests(unittest.TestCase): + def test_model_management_budgets_and_workflow_precedence(self): + defaults = {'context_tokens': '131072', 'output_tokens': '8192', 'safety_tokens': '2048', 'max_tokens': 16384} + self.assertEqual(resolve_settings(defaults, {}), {'context_tokens': 131072, 'output_tokens': 8192, 'safety_tokens': 2048}) + self.assertEqual(resolve_settings(defaults, {'max_tokens': 4096, 'context_tokens': 65536})['output_tokens'], 4096) + self.assertEqual(resolve_settings(defaults, {'output_tokens': 1024})['output_tokens'], 1024) + for params in ({'context_tokens': True}, {'safety_tokens': -1}, {'context_tokens': 4096, 'max_tokens': 8192}): + with self.assertRaises(ValueError): resolve_settings(defaults, params) + + def test_truncated_response_retries_with_original_not_partial_draft(self): + calls, saved, metrics = [], [], {} + def invoke(instruction, content): + calls.append((instruction, content)) + return SimpleNamespace(content='incomplete' if len(calls) == 1 else '# Result\n- fact42', + response_metadata={'finish_reason': 'length' if len(calls) == 1 else 'stop'}) + self.assertIn('fact42', organize('Original fact42', '总结', invoke, checkpoint=saved.append, metrics=metrics)) + self.assertEqual(calls[0][1], calls[1][1]) + self.assertNotIn('incomplete', str(saved)) + self.assertEqual(metrics['compression_retries'], 1) + + def test_complete_overlong_summary_is_semantically_compressed(self): + contents = [] + def invoke(prompt, content): + contents.append(content) + return 'complete verbose result ' * 100 if len(contents) == 1 else '# Summary\n- fact42' + result = organize('Original fact42', '总结', invoke, context_tokens=8192, output_tokens=256) + self.assertIn('fact42', result) + self.assertIn('complete verbose result', contents[1]) + + def test_repeated_truncation_splits_source_and_preserves_completed_sibling(self): + saved, metrics = [], {} + def invoke(prompt, content): + if len(content) > 700: + return SimpleNamespace(content='partial', response_metadata={'finish_reason': 'length'}) + if 'FAIL' in content: + return SimpleNamespace(content='partial', response_metadata={'finish_reason': 'length'}) + return '# Completed42' + with self.assertRaisesRegex(ValueError, '截断'): + organize('A' * 700 + '\nFAIL' + 'B' * 700, '总结', invoke, checkpoint=saved.append, metrics=metrics) + self.assertTrue(any('Completed42' in value for snapshot in saved for value in snapshot)) + self.assertNotIn('partial', str(saved)) + self.assertGreater(metrics['split_retries'], 0) + + def test_echo_model_cannot_grow_merge_forever_or_be_accepted(self): + calls = [] + def invoke(prompt, content): calls.append(1); return content + '\nextra' + with self.assertRaisesRegex(ValueError, '长度'): + organize('facts42 ' * 500, '总结', invoke, context_tokens=8192, output_tokens=256) + self.assertLessEqual(len(calls), 160) + def test_split_preserves_long_unbroken_unicode_and_image_tokens(self): text = '# 标题\n' + '超长段落42' * 8000 + '\nAIHUBIMAGE0001TOKEN\n结束' chunks = split_markdown(text, 2048) @@ -19,7 +68,7 @@ class OrganizeTests(unittest.TestCase): calls.append(content) return SimpleNamespace(content='# Result\n- fact42', response_metadata={}) result = organize(source, '整理事实', invoke, context_tokens=8192, output_tokens=1024, - progress=stages.append, checkpoint=saved.append) + progress=stages.append, checkpoint=saved.append, options={'structure_mode': 'reorganize'}) self.assertIn('fact42', result) self.assertGreater(len(calls), 2) self.assertGreater(len(saved), 2) @@ -39,6 +88,261 @@ class OrganizeTests(unittest.TestCase): return '# saved42' with patch('system_manage.ai_hub_organize.time.sleep'): with self.assertRaisesRegex(ValueError, '已保留') as failure: - organize('word42 ' * 5000, '整理', invoke, context_tokens=4096, output_tokens=512, checkpoint=saved.append) + organize('word42 ' * 5000, '整理', invoke, context_tokens=8192, output_tokens=512, checkpoint=saved.append) self.assertNotIn('private', str(failure.exception)) self.assertEqual(saved, [['# saved42']]) + + +class SummaryPlanningTests(unittest.TestCase): + def test_short_document_one_call_and_language_rules(self): + calls, metrics = [], {} + def invoke(prompt, content): + calls.append((prompt, content)) + return '# Operations\n- Inspect every 42 days.' + result = organize('# Operations\nInspect every 42 days.', '保留事实', invoke, metrics=metrics) + self.assertEqual(len(calls), 1) + self.assertIn('42 days', result) + self.assertIn('ENGLISH ONLY', calls[0][0]) + self.assertIn('Stage: full document', calls[0][0]) + self.assertEqual(metrics['expected_calls'], 1) + self.assertFalse(metrics['needs_merge']) + + def test_budget_counts_prompt_metadata_and_framing(self): + calls, metrics = [], {} + def invoke(prompt, content): + calls.append(1) + self.assertLessEqual(token_bound(prompt) + token_bound(content) + 256, 8192-1024-1024) + return '- fact42' + organize('English data 42. ' * 1500, 'Only facts', invoke, context_tokens=8192, + output_tokens=1024, metrics=metrics) + self.assertEqual(len(calls), metrics['map_chunks']) + self.assertGreater(len(calls), 1) + + def test_output_budget_and_detail_can_require_split(self): + source = '# Operations\n' + 'Inspect equipment 42 days. ' * 180 + brief, detailed = {}, {} + for detail, metric in [('brief', brief), ('detailed', detailed)]: + organize(source, 'Summarize', lambda *a: '- 42 days', output_tokens=1024, + options={'detail': detail}, metrics=metric) + self.assertEqual(brief['map_chunks'], 1) + self.assertGreater(detailed['map_chunks'], 1) + self.assertGreater(detailed['estimated_output_tokens'], brief['estimated_output_tokens']) + + def test_parallel_preserves_order_and_bounds_concurrency(self): + import threading, time + active, peak = 0, 0 + lock = threading.Lock() + source = ''.join(f'## Chapter {i}\n' + 'facts ' * 700 + '\n' for i in range(4)) + def invoke(prompt, content): + nonlocal active, peak + with lock: + active += 1; peak = max(peak, active) + time.sleep(.02 if 'Chapter 0' in content else .005) + with lock: active -= 1 + return '- ' + content[:35].replace('\n', ' ') + metrics = {} + result = organize(source, 'Summarize', invoke, context_tokens=8192, output_tokens=2048, + options={'concurrency': 2}, metrics=metrics) + self.assertEqual(peak, 2) + self.assertEqual(metrics['model_calls'], metrics['map_chunks']) + self.assertLess(result.index('Chapter 0'), result.index('Chapter 1')) + + def test_failure_resume_reuses_only_matching_configuration(self): + source = 'facts42 ' * 1200 + states, calls = [], [] + def fail(prompt, content): + calls.append(1) + if len(calls) > 1: raise RuntimeError('secret') + return '- retained42' + with self.assertRaises(ValueError): + organize(source, 'Summarize', fail, context_tokens=8192, output_tokens=1024, + options={'retries': 0}, state_checkpoint=states.append) + metrics = {} + organize(source, 'Summarize', lambda *a: '- resumed42', context_tokens=8192, output_tokens=1024, + options={'retries': 0}, resume=states[-1], metrics=metrics) + self.assertEqual(metrics['reused_chunks'], 1) + self.assertEqual(metrics['model_calls'], metrics['map_chunks']-1) + other = {} + organize(source, 'New prompt', lambda *a: '- new42', context_tokens=8192, output_tokens=1024, + options={'retries': 0}, resume=states[-1], metrics=other) + self.assertEqual(other['reused_chunks'], 0) + + def test_chunk_metadata_and_mixed_language_survive_merge(self): + seen = [] + def invoke(prompt, content): + seen.append(prompt) + return '# Maintenance\n- 42 days\n## 检修\n- 每月一次' if '检修' in content else '# Maintenance\n- 42 days' + organize('## Maintenance 检修\n' + '42 days ' * 1400, 'Summarize', invoke, + context_tokens=8192, output_tokens=2048, options={'structure_mode': 'reorganize'}) + self.assertTrue(any('batch' in p and 'character range' in p and 'ancestor path' in p for p in seen)) + self.assertTrue(any('全局汇总阶段' in p for p in seen)) + self.assertTrue(all('original' in p.lower() for p in seen)) + + def test_no_split_option_rejects_before_call(self): + invoke = unittest.mock.Mock() + with self.assertRaisesRegex(ValueError, '开启自动拆分'): + organize('facts ' * 5000, 'Summarize', invoke, options={'auto_split': False}) + invoke.assert_not_called() + + def test_invalid_settings_and_declared_model_limits(self): + from system_manage.summary_plan import validate_options + for options in ({'concurrency': True}, {'top_p': float('nan')}, {'detail': 'unknown'}, + {'parallel': 'true'}, {'context_tokens': 1024}, {'max_depth': 7}): + with self.assertRaises(ValueError): validate_options(options) + with self.assertRaisesRegex(ValueError, '上限'): + resolve_settings({'context_tokens': 8192}, {'context_tokens': 16384}) + with self.assertRaises(ValueError): + resolve_settings({'max_output_tokens': 1024}, {'max_tokens': 2048}) + + def test_max_input_independent_limit(self): + stats = {} + organize('data42 ' * 2000, 'Summarize', lambda *a: '- data42', output_tokens=512, + options={'max_input_tokens': 8192}, metrics=stats) + self.assertLess(stats['input_budget'], 8192) + self.assertGreater(stats['map_chunks'], 1) + + + def test_table_list_and_fenced_example_stay_whole_when_they_fit(self): + table = '| Key | Value |\n| --- | --- |\n| Count | 42 |\n' + code = '```text\n# not a heading\nvalue42\n```\n' + items = '- step 1\n- step 2\n' + for block in (table, code, items): + text = 'Intro ' * 45 + '\n' + block + 'After ' * 60 + parts = split_markdown(text, 300) + self.assertEqual(''.join(parts), text) + self.assertTrue(any(block in part for part in parts)) + self.assertTrue(all(token_bound(part) <= 300 for part in parts)) + + def test_custom_node_length_changes_output_planning(self): + metrics = [] + for length in (20, 120): + stats = {} + organize('Equipment 42 days. ' * 240, 'Summarize', lambda *a: '- 42 days', output_tokens=1024, + options={'node_max_chars': length}, metrics=stats) + metrics.append(stats) + self.assertGreater(metrics[1]['map_chunks'], metrics[0]['map_chunks']) + + def test_provider_usage_prevents_unnecessary_compression_from_byte_estimate(self): + calls = [] + def invoke(*a): + calls.append(1) + return SimpleNamespace(content='中文结论42。' * 30, response_metadata={'finish_reason': 'stop'}, + usage_metadata={'output_tokens': 200}) + organize('原文42', 'Summarize', invoke, output_tokens=256) + self.assertEqual(len(calls), 1) + + + def test_wrong_language_retries_only_current_source_and_unwraps_any_fence(self): + contents, metrics = [], {} + def invoke(prompt, content): + contents.append(content) + return '中文错误结果' if len(contents) == 1 else '```ruby\n# Maintenance\n- Inspect every 42 days.' + output = organize('Inspect every 42 days.', 'Summarize', invoke, metrics=metrics) + self.assertEqual(len(contents), 2) + self.assertEqual(contents[0], contents[1]) + self.assertEqual(output, '# Maintenance\n- Inspect every 42 days.') + self.assertEqual(metrics['language_retries'], 1) + + def test_repeated_wrong_language_is_not_success(self): + with self.assertRaisesRegex(ValueError, '未保持原文语言'): + organize('Inspect every 42 days.', 'Summarize', lambda *a: '中文错误结果') + + def test_language_detection_preserves_source_despite_translation_prompt(self): + from system_manage.summary_plan import source_language + self.assertEqual(source_language('English text', '请翻译为中文'), 'english') + self.assertEqual(source_language('これは日本語の説明です'), 'japanese') + self.assertEqual(source_language('한국어 설명입니다'), 'korean') + + + def test_original_headings_are_validated_and_levels_restored(self): + calls=[] + def invoke(*a): + calls.append(1) + return '- fact42' if len(calls)==1 else '### Maintenance\n#### Inspection\n- fact42' + result=organize('# Maintenance\n## Inspection\nInspect every 42 days.', 'Summarize', invoke, + options={'structure_mode':'preserve'}) + self.assertTrue(result.startswith('# Maintenance\n## Inspection')) + self.assertEqual(len(calls),2) + + def test_ollama_length_stop_is_never_accepted(self): + with self.assertRaisesRegex(ValueError,'截断'): + organize('English source', 'Summarize', lambda *a: SimpleNamespace(content='Partial', + response_metadata={'done_reason':'length','eval_count':256}),output_tokens=256) + + + def test_mixed_language_failure_recovers_only_language_local_sections(self): + calls, stats = [], {} + source='# Maintenance / 维护\n## English operations\nInspect every 42 days.\n## 中文操作\n每31天记录一次运行情况。' + def invoke(prompt, content): + calls.append(content) + if len(calls)==1:return '## 英文操作\n每42天检查一次。\n## 中文操作\n每31天记录一次运行情况。' + if 'English operations' in content:return '## English operations\n- Inspect every 42 days.' + return '## 中文操作\n- 每31天记录一次运行情况。' + output=organize(source,'Summarize',invoke,options={'structure_mode':'preserve'},metrics=stats) + self.assertEqual(len(calls),3) + self.assertNotIn('中文操作',calls[1]) + self.assertNotIn('English operations',calls[2]) + self.assertIn('42 days',output) + self.assertIn('每31天',output) + self.assertEqual(stats['language_splits'],1) + self.assertEqual(stats['expected_calls'],3) + + def test_single_body_heading_chain_is_restored_without_another_call(self): + calls=[] + def invoke(*a):calls.append(1);return '### Maintenance\n- Inspect every 42 days.' + output=organize('# Maintenance\n## Inspection\nInspect every 42 days.','Summarize',invoke, + options={'structure_mode':'preserve'}) + self.assertEqual(len(calls),1) + self.assertIn('# Maintenance\n## Inspection',output) + + def test_customized_default_prompt_is_not_discarded(self): + from system_manage.summary_plan import DEFAULT_SUMMARY_PROMPT,prompt_rules + custom=DEFAULT_SUMMARY_PROMPT+'\nKeep the STOP condition explicit.' + self.assertIn(custom,prompt_rules(custom,{},'english')) + + + def test_independent_chapter_does_not_inherit_previous_sibling_context(self): + seen=[] + source='## Alpha\n'+'Alpha fact42. '*180+'\n## Beta\n'+'Beta fact43. '*180 + def invoke(prompt,content): + seen.append((prompt,content));return '- fact42 and fact43' + organize(source,'Summarize',invoke,context_tokens=8192,output_tokens=1024) + beta=[prompt for prompt,content in seen if content.startswith('## Beta')] + self.assertTrue(beta) + self.assertTrue(all('ancestor path: document start' in prompt for prompt in beta)) + self.assertTrue(all('Previous source excerpt' not in prompt for prompt in beta)) + + + def test_model_wrapper_and_trailing_self_comment_are_removed_without_call(self): + for wrapped in ('```markdown\n# Result\n- fact42\n```\nThis summary is complete.', + '# Result\n- fact42\n```\nThis summary is complete.'): + calls=[] + def invoke(*a):calls.append(1);return wrapped + result=organize('Original fact42','Summarize',invoke) + self.assertEqual(result,'# Result\n- fact42') + self.assertEqual(len(calls),1) + + + def test_explicit_numeric_conditions_keep_strictness_negation_and_sign(self): + from system_manage.summary_plan import boundaries_match + self.assertTrue(boundaries_match('Stop above 91 degrees.', 'Stop when temperature exceeds 91.0 degrees.')) + self.assertFalse(boundaries_match('Stop above 91 degrees.', 'Stop when it reaches 91 degrees.')) + self.assertFalse(boundaries_match('温度超过90℃停止。', '温度达到90℃停止。')) + self.assertTrue(boundaries_match('Do not exceed 90.', 'At most 90.0.')) + self.assertFalse(boundaries_match('At least -5.', 'Below -5.')) + self.assertTrue(boundaries_match('Thunder 42.', 'Thunder 42.')) + + def test_changed_numeric_boundary_repairs_from_original_only(self): + contents,stats=[],{} + def invoke(prompt,content): + contents.append(content) + return '- Stop when it reaches 91 degrees.' if len(contents)==1 else '- Stop above 91 degrees.' + output=organize('Stop above 91 degrees.','Summarize',invoke,metrics=stats) + self.assertEqual(len(contents),2) + self.assertEqual(contents[0],contents[1]) + self.assertIn('above 91',output) + self.assertEqual(stats['fact_retries'],1) + + def test_repeated_numeric_boundary_change_cannot_be_success(self): + with self.assertRaisesRegex(ValueError,'数值边界条件'): + organize('Stop above 91 degrees.','Summarize',lambda *a:'- Stop when it reaches 91 degrees.') diff --git a/apps/system_manage/urls.py b/apps/system_manage/urls.py index 72cbea963..5c3ad23da 100644 --- a/apps/system_manage/urls.py +++ b/apps/system_manage/urls.py @@ -1,13 +1,15 @@ +from system_manage.views.mcp_engine import EngineView from django.urls import path from . import views -from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport, DocumentExport +from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport, DocumentExport, ServiceCalls from .views.platform_access import AccessView, RoleView, GrantsView, UserAccessView from .views.access_control_policy import AccessControlRuntimeView, AccessControlDecisionView app_name = "system_manage" # @formatter:off urlpatterns = [ + path('workspace//mcp_engine/', EngineView.as_view()), path('platform_access', AccessView.as_view()), path('platform_access/roles', RoleView.as_view()), path('platform_access/roles/', RoleView.as_view()), @@ -18,6 +20,7 @@ urlpatterns = [ path('workspace//ai_hub/exports', MarkdownExport.as_view()), path('workspace//ai_hub/assets//content', AssetContent.as_view()), path('workspace//ai_hub/parsers//test', ParserTest.as_view()), + path('workspace//ai_hub/services//calls', ServiceCalls.as_view()), path('workspace//ai_hub/published//run', PublishedRun.as_view()), path('workspace//ai_hub/', HubView.as_view()), path('workspace//ai_hub//', HubView.as_view()), diff --git a/apps/system_manage/views/ai_hub.py b/apps/system_manage/views/ai_hub.py index 9889b16bb..904640641 100644 --- a/apps/system_manage/views/ai_hub.py +++ b/apps/system_manage/views/ai_hub.py @@ -35,10 +35,18 @@ class HubView(APIView): raise PermissionDenied('未授权使用思维导图') def get(self, request, workspace_id, kind, pk=None): + if kind == 'artifact_limit' and not pk: + from system_manage import artifact_settings + hub.require_admin(request.user) + return result.success({'limit': artifact_settings.limit()}) + if kind == 'workbench_defaults' and not pk: + from system_manage import workbench_defaults + return result.success(workbench_defaults.read(workspace_id, request.user)) if kind == 'parser_catalog': from system_manage.parsers.catalog import catalogue return result.success(catalogue()) if kind == 'resources': + from system_manage.mcp_business import summary_model_metadata parsers = hub.visible(hub.AIParserResource.objects.filter(workspace_id=workspace_id, enabled=True), request.user) from users.models import User from users.models.user_manage_ext import UserActivationApplication @@ -48,7 +56,9 @@ class HubView(APIView): resource_type__in=['PARSER', 'SERVICE'], actions__contains=['AUTHORIZE']).exists() return result.success({'parsers': [hub.serialize(p, user=request.user) for p in parsers if 'USE' in hub.serialize(p, user=request.user)['actions']], - 'models': list(hub.authorized_models(workspace_id, request.user).values('id', 'name', 'model_type')), + 'models': [summary_model_metadata(model) + if model.model_type == 'LLM' else {'id': str(model.id), 'name': model.name, 'model_type': model.model_type} + for model in hub.authorized_models(workspace_id, request.user)], 'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if may_authorize else [], 'is_admin': hub.admin(request.user)}) if pk: @@ -61,9 +71,33 @@ class HubView(APIView): raise ValidationError('资源类型无效') query = hub.visible(hub.KINDS[kind].objects.filter(workspace_id=workspace_id), request.user) if kind == 'tasks': + from django.db.models import Q + query = query.filter(Q(snapshot__engine__isnull=True) | Q(snapshot__engine=False)) query = query.defer('file_data', 'input_data', 'snapshot', 'result') elif kind == 'artifacts': query = query.defer('original', 'markdown', 'history') + if any(key in request.query_params for key in ('mine', 'current_page', 'page_size', 'name')): + mine = request.query_params.get('mine', 'false') + if mine not in ('true', 'false'): + raise ValidationError('查询参数无效') + try: + page = int(request.query_params.get('current_page', '1')) + size = int(request.query_params.get('page_size', '20')) + if page < 1 or size < 1 or size > 100: + raise ValueError() + except (ValueError, TypeError): + raise ValidationError('分页参数无效') + name = request.query_params.get('name', '') + if len(name) > 128: + raise ValidationError('名称无效') + if mine == 'true': + query = query.filter(owner_id=request.user.id) + if name: + query = query.filter(name__icontains=name) + total = query.count() + records = query.order_by('-updated_at', '-id')[(page - 1) * size:page * size] + return result.success({'records': [hub.serialize(obj, user=request.user) for obj in records], + 'total': total, 'current': page}) return result.success([hub.serialize(obj, user=request.user) for obj in query.order_by('-created_at')[:200]]) def post(self, request, workspace_id, kind, pk=None): @@ -76,6 +110,20 @@ class HubView(APIView): raise ValidationError('任务参数无效') if not isinstance(data, dict): raise ValidationError('请求参数无效') + if kind == 'artifact_limit' and not pk: + from system_manage import artifact_settings + return result.success(artifact_settings.save(request.user, data)) + if kind == 'workbench_defaults' and not pk: + from system_manage import workbench_defaults + return result.success(workbench_defaults.save(workspace_id, request.user, data)) + if kind == 'summary_preview': + from system_manage.mcp_business import summary_preview + from system_manage.access_control import enforce_task + enforce_task(request, nodes=data.get('nodes')) + return result.success(summary_preview(workspace_id, request.user, data)) + if kind == 'heading_structure': + from system_manage.heading_service import process + return result.success(process(workspace_id, request.user, data, request)) if kind == 'tasks': from system_manage.access_control import enforce_task if pk: @@ -97,7 +145,12 @@ class HubView(APIView): if kind == 'parsers': obj = hub.parser_save(workspace_id, request.user, data, obj) elif kind == 'artifacts': - obj = hub.artifact_save(workspace_id, request.user, data, obj) + if 'action' in data: + if data['action'] != 'rename': + raise ValidationError('操作无效') + obj = hub.artifact_rename(obj, data) + else: + obj = hub.artifact_save(workspace_id, request.user, data, obj) elif kind == 'services': if set(data) - {'allowed_users'}: hub.require_resource_action(request.user, obj, 'EDIT') @@ -120,12 +173,17 @@ class HubView(APIView): if data.get('revoke_key'): obj.key_hash = '' obj.save() + if 'api_config' in data: + from system_manage.ai_hub_service_api import save_config + save_config(obj.id, data['api_config']) output = hub.serialize(obj) if key: output['api_key'] = key return result.success(output) elif kind == 'tasks': obj = AITask.objects.select_for_update().get(pk=obj.pk) + if obj.snapshot.get('engine'): + raise ValidationError('请在 MCP 引擎任务页重试该任务') if obj.status != 'FAILED': raise ValidationError('仅失败任务可重试') if hub.make_snapshot(workspace_id, request.user, obj.snapshot['nodes']) != obj.snapshot['nodes']: @@ -151,6 +209,8 @@ class HubView(APIView): raise ValidationError('待处理任务已达上限') obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), request=request) elif kind == 'artifacts': + if 'action' in data: + raise ValidationError('操作无效') obj = hub.artifact_save(workspace_id, request.user, data) else: raise ValidationError('资源类型无效') @@ -225,7 +285,79 @@ class MarkdownExport(HubView): class PublishedRun(HubView): """Internal service: login AND explicit resource grants; Key is an optional extra credential.""" authentication_classes = [TokenAuth] - http_method_names = ['post', 'options'] + http_method_names = ['get', 'post', 'options'] + + def handle_exception(self, exc): + from rest_framework.exceptions import APIException + from rest_framework.response import Response + from common.exception.app_exception import AppApiException + if isinstance(exc, (APIException, AppApiException)): + status = exc.status_code + defaults = {400: '请求参数无效', 401: '请先登录', 403: '无权调用该服务', + 404: '服务不存在或无权访问', 405: '不支持的请求方法', 429: '调用过于频繁'} + message = defaults.get(status, '调用失败') + detail = getattr(exc, 'detail', None) + while isinstance(detail, (list, tuple, dict)) and detail: + detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0] + safe_messages = {'服务Key无效', '调用过于频繁', '请求方法与服务配置不一致', '请求参数过长', + '任务参数无效', '包含未知参数', '参数不能重复', '文件参数无效', '参数类型无效', + '缺少必填参数', '优先级无效', '服务已停用', '请选择文件', '文档内容不能为空', + '文件任务需要解析节点', '待处理任务已达上限', '文件不能超过20MB'} + if str(detail) in safe_messages: + message = str(detail) + self._call_error = message + return Response({'code': getattr(exc, 'code', status), 'message': message}, status=status) + raise exc + + def dispatch(self, request, *args, **kwargs): + import time + self._call_started = time.monotonic() + try: + with transaction.atomic(): + return super().dispatch(request, *args, **kwargs) + except Exception: + import logging + from system_manage.models import Log + logging.getLogger(__name__).error('AI service call failed') + try: + Log.objects.create(menu='AI_SERVICE', operate='CALL', workspace_id=str(kwargs.get('workspace_id', '')), + operation_object={'service_id': str(kwargs.get('pk', ''))}, user={}, status=500, ip_address='', + details={'method': request.method, 'http_status': 500, + 'elapsed_ms': max(0, int((time.monotonic() - self._call_started) * 1000)), + 'status': 'FAILED', 'error': '服务执行失败', 'task_id': None}) + except Exception: + return result.error('调用日志保存失败', response_status=503) + return result.error('服务执行失败', response_status=500) + + def finalize_response(self, request, response, *args, **kwargs): + import time + from system_manage.models import Log + from rest_framework.response import Response + response = super().finalize_response(request, response, *args, **kwargs) + body = getattr(response, 'data', None) + if body is None and response.get('Content-Type', '').startswith('application/json'): + import json + try: + body = json.loads(response.content) + except (ValueError, TypeError): + body = None + code = body.get('code') if isinstance(body, dict) else None + success = response.status_code < 400 and code in (None, 0, 200) + try: + Log.objects.create(menu='AI_SERVICE', operate='CALL', workspace_id=str(kwargs.get('workspace_id', '')), + operation_object={'service_id': str(kwargs.get('pk', ''))}, + user={'id': str(getattr(getattr(request, '_user', None), 'id', '') or '')}, status=response.status_code if success or response.status_code >= 400 else 400, ip_address='', + details={'method': request.method, 'http_status': response.status_code, + 'elapsed_ms': max(0, int((time.monotonic() - self._call_started) * 1000)), + 'status': 'SUCCESS' if success else 'FAILED', 'error': '' if success else getattr(self, '_call_error', '调用失败'), + 'task_id': getattr(self, '_task_id', None) if success else None}) + except Exception: + transaction.set_rollback(True) + response = super().finalize_response(request, Response({'code': 503, 'message': '调用日志保存失败'}, status=503), *args, **kwargs) + return response + + def get(self, request, workspace_id, pk): + return self.post(request, workspace_id, pk) def post(self, request, workspace_id, pk): service = hub.one('services', pk, workspace_id, request.user) @@ -241,17 +373,46 @@ class PublishedRun(HubView): cache.add(key, 0, 70) if cache.incr(key) > service.requests_per_minute: raise ValidationError('调用过于频繁') - data = request.data - if request.FILES: - import json - try: - data = json.loads(request.data.get('data', '{}')) - except (ValueError, TypeError): - raise ValidationError('任务参数无效') - task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service, request=request) + from system_manage.ai_hub_service_api import get_config, parse_request + data, upload = parse_request(request, get_config(service.id)) + task = hub.submit(workspace_id, request.user, data, upload, service=service, request=request) + self._task_id = str(task.id) return result.success(hub.serialize(task)) +class ServiceCalls(HubView): + http_method_names = ['get', 'options'] + + def get(self, request, workspace_id, pk): + from system_manage.models import Log + service = hub.one('services', pk, workspace_id, request.user) + hub.require_resource_action(request.user, service, 'EDIT') + try: + page = int(request.query_params.get('current_page', 1)) + size = int(request.query_params.get('page_size', 20)) + if page < 1 or size < 1 or size > 100: + raise ValueError() + except (ValueError, TypeError): + raise ValidationError('分页参数无效') + logs = Log.objects.filter(menu='AI_SERVICE', operate='CALL', workspace_id=workspace_id, + operation_object__service_id=str(pk)).order_by('-create_time') + total = logs.count() + rows = list(logs[(page - 1) * size:page * size]) + ids = [row.details.get('task_id') for row in rows if row.details.get('task_id')] + tasks = {str(t.id): t for t in AITask.objects.filter(id__in=ids, workspace_id=workspace_id)} + records = [] + for row in rows: + details = row.details + task = tasks.get(details.get('task_id')) + duration = int((task.finished_at - task.started_at).total_seconds() * 1000) if task and task.started_at and task.finished_at else None + records.append({'id': str(row.id), 'created_at': row.create_time.isoformat(), + 'method': details.get('method'), 'status': details.get('status'), + 'http_status': details.get('http_status'), 'elapsed_ms': details.get('elapsed_ms'), + 'error': (task.error if task and task.status == 'FAILED' else details.get('error')), 'task_id': details.get('task_id'), + 'task_status': task.status if task else None, 'execution_ms': duration}) + return result.success({'records': records, 'total': total}) + + class DocumentExport(HubView): http_method_names = ['post', 'options'] @@ -259,6 +420,8 @@ class DocumentExport(HubView): from django.http import HttpResponse from system_manage.document_export import export_document, ExportError, MAX_HTML from system_manage.ai_hub_assets import ASSET, image_bytes + import time + started = time.perf_counter() def image(src): match = ASSET.fullmatch(src) if not match or match[1] != workspace_id: @@ -275,6 +438,7 @@ class DocumentExport(HubView): except (ExportError, ValueError, TypeError) as exc: raise ValidationError(str(exc) if isinstance(exc, ExportError) else '文档内容格式无效') from exc response = HttpResponse(raw, content_type=mime) + response['Server-Timing'] = f'document_export;dur={(time.perf_counter() - started) * 1000:.1f}' response['Content-Disposition'] = 'attachment; filename="document.' + format_name + '"' response['Cache-Control'] = 'private, no-store' response['X-Content-Type-Options'] = 'nosniff' diff --git a/scripts/ai_hub_ops.py b/scripts/ai_hub_ops.py index 906432eb0..8e2653410 100644 --- a/scripts/ai_hub_ops.py +++ b/scripts/ai_hub_ops.py @@ -9,6 +9,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=['check', 'plan', 'migrate', 'worker', 'cleanup-assets'], nargs='?', default='check') parser.add_argument('--apply', action='store_true', help='Apply expired image cleanup (default: report only)') + parser.add_argument('--concurrency', type=int, default=1) parser.add_argument('--once', action='store_true') parser.add_argument('--recover', action='store_true') args = parser.parse_args() @@ -35,7 +36,7 @@ def main(): elif args.action == 'cleanup-assets': call_command('ai_hub_cleanup_assets', apply=args.apply) elif args.action == 'worker': - call_command('ai_hub_worker', once=args.once, recover=args.recover) + call_command('ai_hub_worker', once=args.once, recover=args.recover, concurrency=args.concurrency) else: from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact tables = set(connection.introspection.table_names()) diff --git a/ui/src/api/ai-hub.ts b/ui/src/api/ai-hub.ts index cfa3da5ff..768d9b8b7 100644 --- a/ui/src/api/ai-hub.ts +++ b/ui/src/api/ai-hub.ts @@ -29,5 +29,6 @@ export async function hubBinary(kind: 'assets' | 'exports' | 'document_export', try { const detail = JSON.parse(await response.data.text()); if (typeof detail.message === 'string') message = detail.message.slice(0, 200) } catch { /* retain a safe fallback */ } throw new Error(message) } - return { content: await response.data.arrayBuffer(), mime: response.data.type } + return { content: await response.data.arrayBuffer(), mime: response.data.type, + serverTimings: response.headers?.['server-timing'] || '' } } diff --git a/ui/src/components/ai-chat/component/knowledge-source-component/ExecutionDetailCard.vue b/ui/src/components/ai-chat/component/knowledge-source-component/ExecutionDetailCard.vue index 2b6e4af57..4e9e73453 100644 --- a/ui/src/components/ai-chat/component/knowledge-source-component/ExecutionDetailCard.vue +++ b/ui/src/components/ai-chat/component/knowledge-source-component/ExecutionDetailCard.vue @@ -38,6 +38,7 @@
+