225 lines
13 KiB
Python
225 lines
13 KiB
Python
"""Queue published advanced workflows without changing interactive chat sessions."""
|
|
import json
|
|
import uuid
|
|
from copy import deepcopy
|
|
from types import SimpleNamespace
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
from rest_framework.exceptions import ValidationError, NotFound, PermissionDenied
|
|
from system_manage import ai_hub as hub, mcp_registry as registry
|
|
from system_manage.models.ai_hub import AITask
|
|
|
|
|
|
def application_for(workspace, user, pk):
|
|
from application.models import Application
|
|
from rest_framework.serializers import UUIDField
|
|
obj = Application.objects.filter(pk=UUIDField().run_validation(pk), workspace_id=workspace, type='WORK_FLOW').first()
|
|
if not obj:
|
|
raise NotFound('高级编排应用不存在')
|
|
if not hub.admin(user) and str(obj.user_id) != str(user.id):
|
|
from system_manage.platform_access import resource_actions
|
|
if 'USE' not in resource_actions(user, 'APPLICATION', obj.id, workspace):
|
|
raise PermissionDenied('未授权使用该应用流程')
|
|
return obj
|
|
|
|
|
|
def nodes(graph):
|
|
for node in graph.get('nodes', []):
|
|
yield node
|
|
body = node.get('properties', {}).get('node_data', {}).get('loop_body')
|
|
if isinstance(body, dict):
|
|
yield from nodes(body)
|
|
|
|
|
|
def requirements(graph):
|
|
result = set()
|
|
for node in nodes(graph):
|
|
data = node.get('properties', {}).get('node_data', {}) or {}
|
|
kind = node.get('type')
|
|
if kind == 'component-node':
|
|
result.add(data.get('component_id', ''))
|
|
elif kind == 'mcp-node':
|
|
result.add('mcp:' + data['mcp_tool_id'] if data.get('mcp_source') == 'referencing' else 'mcp-custom')
|
|
elif kind in ('ai-chat-node', 'question-node', 'intent-node') and data.get('model_id'):
|
|
result.add('model:' + str(data['model_id']))
|
|
result.update((data.get('model_setting') or {}).get('engine_tools', []))
|
|
if data.get('mcp_enable'):
|
|
ids = list(data.get('mcp_tool_ids') or [])
|
|
if data.get('mcp_tool_id'): ids.append(data['mcp_tool_id'])
|
|
result.update('mcp:' + str(key) for key in ids)
|
|
if data.get('mcp_source') == 'custom' and data.get('mcp_servers'):
|
|
result.add('mcp-custom')
|
|
elif kind == 'document-extract-node':
|
|
result.add('file-reader')
|
|
return sorted(result)
|
|
|
|
|
|
def knowledge_scope(graph):
|
|
if hasattr(graph, 'nodes'):
|
|
graph = {'nodes': [{'type': node.type, 'properties': node.properties} for node in graph.nodes]}
|
|
ids = set()
|
|
for node in nodes(graph):
|
|
data = node.get('properties', {}).get('node_data', {}) or {}
|
|
ids.update(str(key) for key in data.get('knowledge_id_list', []))
|
|
if data.get('component_id') == 'knowledge-search' and data.get('sources', {}).get('knowledge_ids') != 'reference':
|
|
ids.update(str(key) for key in data.get('parameters', {}).get('knowledge_ids', []))
|
|
return sorted(ids)
|
|
|
|
|
|
def workbench_flows(workspace, user):
|
|
from application.models import Application, ApplicationVersion
|
|
from system_manage.platform_access import resource_actions
|
|
rows = []
|
|
for app in Application.objects.filter(workspace_id=workspace, type='WORK_FLOW', is_publish=True).order_by('-update_time')[:200]:
|
|
if not hub.admin(user) and str(app.user_id) != str(user.id) and 'USE' not in resource_actions(user, 'APPLICATION', app.id, workspace):
|
|
continue
|
|
version = ApplicationVersion.objects.filter(application=app).order_by('-create_time', '-id').first()
|
|
if not version or any(node.get('type') == 'form-node' for node in nodes(version.work_flow)):
|
|
continue
|
|
if any(node.get('type') == 'document-extract-node' for node in nodes(version.work_flow)):
|
|
continue
|
|
required = requirements(version.work_flow)
|
|
if not {'mindmap-engine', 'mindmap-save'} & set(required):
|
|
continue
|
|
rows.append({'id': str(app.id), 'name': app.name, 'version': str(version.id),
|
|
'with_file': any(key == 'file-reader' or key.startswith('parser:') for key in required)})
|
|
return rows
|
|
|
|
|
|
@transaction.atomic
|
|
def submit(workspace, user, data, upload=None):
|
|
from users.models import User
|
|
from application.models import ApplicationVersion
|
|
User.objects.select_for_update().get(pk=user.id)
|
|
app = application_for(workspace, user, data.get('application_id'))
|
|
version = ApplicationVersion.objects.filter(application=app).order_by('-create_time', '-id').first()
|
|
if not app.is_publish or not version:
|
|
raise ValidationError('请先在高级编排中发布流程版本')
|
|
graph = deepcopy(version.work_flow)
|
|
if any(n.get('type') == 'form-node' for n in nodes(graph)):
|
|
raise ValidationError('后台任务不支持等待用户填写表单,请在问答页面运行该流程')
|
|
required = requirements(graph)
|
|
save_output = data.get('save_output', False)
|
|
if not isinstance(save_output, bool):
|
|
raise ValidationError('保存图谱选项无效')
|
|
if save_output:
|
|
from system_manage.platform_access import module_list
|
|
if 'mindmap' not in module_list(user):
|
|
raise PermissionDenied('未授权使用思维导图')
|
|
if not {'mindmap-engine', 'mindmap-save'} & set(required):
|
|
raise ValidationError('该流程未配置导图组件')
|
|
required = sorted(set([*required, 'mindmap-save']))
|
|
versions = {key: registry.resolve(workspace, user, key)[0]['version'] for key in required}
|
|
if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
|
|
raise ValidationError('待处理任务已达上限')
|
|
if upload and upload.size > hub.MAX_FILE:
|
|
raise ValidationError('文件不能超过20MB')
|
|
return AITask.objects.create(workspace_id=workspace, owner_id=user.id,
|
|
name=hub.text(data.get('name') or app.name, '任务名称', limit=128),
|
|
snapshot={'engine': True, 'dispatch': 'worker', 'application_id': str(app.id),
|
|
'application_version': str(version.id), 'work_flow': graph, 'resources': required,
|
|
'resource_versions': versions, 'save_output': save_output,
|
|
'source': hub.text(data.get('source', 'advanced-workflow'), '任务来源', limit=80),
|
|
'priority': hub.integer(data.get('priority', 5), 0, 9, '优先级')},
|
|
input_data={'question': hub.text(data.get('question', ''), '输入内容', required=False)},
|
|
file_name=upload.name[:255] if upload else '', file_data=upload.read() if upload else None)
|
|
|
|
|
|
def execute(task):
|
|
from users.models import User
|
|
from application.flow.common import Workflow
|
|
from application.flow.workflow_manage import WorkflowManage
|
|
from system_manage import mcp_scheduler as scheduler
|
|
user = User.objects.get(pk=task.owner_id, is_active=True)
|
|
app = application_for(task.workspace_id, user, task.snapshot['application_id'])
|
|
for key, version in task.snapshot.get('resource_versions', {}).items():
|
|
if registry.resolve(task.workspace_id, user, key)[0]['version'] != version:
|
|
raise ValidationError('组件版本已变化,请重新提交流程任务')
|
|
chat_id, record_id = str(uuid.uuid4()), str(uuid.uuid4())
|
|
|
|
class Handler:
|
|
chat_info = SimpleNamespace(application=app, debug=True, application_id=str(app.id),
|
|
chat_id=chat_id, get_chat_variable=lambda: {})
|
|
|
|
def handler(self, workflow):
|
|
details = workflow.get_runtime_details()
|
|
answers = workflow.get_answer_text_list()
|
|
text = '\n\n'.join(item.get('content', '') for group in answers for item in group)
|
|
status = 'SUCCESS' if workflow.status == 200 else 'FAILED'
|
|
AITask.objects.filter(pk=task.id, attempt=task.attempt, status='RUNNING').update(
|
|
steps=list(details.values()), updated_at=timezone.now())
|
|
task.refresh_from_db(fields=['result'])
|
|
if status == 'SUCCESS' and task.snapshot.get('save_output') and not task.result.get('artifact_id'):
|
|
from system_manage.mcp_runtime import invoke
|
|
invoke(task.workspace_id, user, 'mindmap-save',
|
|
{'text': task.result.get('mindmap_markdown') or text, 'name': task.file_name or task.name},
|
|
parent=task.id, application_id=str(app.id), scope={'source': 'mindmap-workflow'})
|
|
task.refresh_from_db(fields=['result'])
|
|
scheduler.finish(task, status, {**task.result, 'text': text, 'answers': answers}, '' if status == 'SUCCESS' else '流程节点执行失败,请查看执行详情')
|
|
|
|
workflow = Workflow.new_instance(deepcopy(task.snapshot['work_flow']))
|
|
params = {'history_chat_record': [], 'question': task.input_data.get('question') or '处理已上传文档',
|
|
'chat_id': chat_id, 'chat_record_id': record_id, 'stream': False,
|
|
'workspace_id': task.workspace_id, 'application_id': str(app.id), 're_chat': False,
|
|
'debug': True, 'chat_user_id': str(user.id), 'chat_user_type': 'PLATFORM_USER',
|
|
'engine_task_id': str(task.id), 'engine_user_id': str(user.id)}
|
|
class TaskWorkflow(WorkflowManage):
|
|
def run_chain_manage(self, *args, **kwargs):
|
|
from django.db import connection
|
|
try:
|
|
return super().run_chain_manage(*args, **kwargs)
|
|
finally:
|
|
connection.close()
|
|
def record_node(self, node, state):
|
|
with transaction.atomic():
|
|
current = AITask.objects.select_for_update().get(pk=task.id)
|
|
entry = {'node_id': node.id, 'name': node.node.properties.get('stepName'),
|
|
'type': node.type, 'status': state, 'runtime_node_id': node.runtime_node_id}
|
|
values = [item for item in current.steps if item.get('runtime_node_id') != node.runtime_node_id]
|
|
current.steps = [*values, entry]
|
|
current.save(update_fields=['steps', 'updated_at'])
|
|
def run_node(self, node):
|
|
self.record_node(node, 'RUNNING')
|
|
try:
|
|
result = super().run_node(node)
|
|
self.record_node(node, 'SUCCESS')
|
|
return result
|
|
except Exception:
|
|
self.record_node(node, 'FAILED')
|
|
raise
|
|
manager = TaskWorkflow(workflow, params, Handler())
|
|
manager.run()
|
|
task.refresh_from_db()
|
|
return task
|
|
|
|
|
|
def new_template(workspace, user, data):
|
|
"""A normal advanced workflow, editable and versioned by the existing editor."""
|
|
from application.models import Application
|
|
hub.require_admin(user)
|
|
def node(key, kind, name, x, y, fields=None, values=None):
|
|
return {'id': key, 'type': kind, 'x': x, 'y': y, 'properties': {'stepName': name,
|
|
'config': {'fields': fields or [], 'globalFields': []}, 'user_input_field_list': [], 'api_input_field_list': [], 'input_field_list': [], 'node_data': values or {}}}
|
|
fields = [{'label': '文本结果', 'value': 'text'}, {'label': '结构化结果', 'value': 'structured'}]
|
|
upload = data.get('with_file', False)
|
|
items = [node('base-node', 'base-node', '基本信息', -500, 380),
|
|
node('start-node', 'start-node', '开始', 100, 380, [{'label': '用户问题', 'value': 'question'}, {'label': '文档', 'value': 'document'}])]
|
|
items[0]['properties']['node_data'] = {'name': data.get('name', '文档处理流程'), 'desc': '', 'prologue': '请上传文档或输入内容。', 'tts_type': 'BROWSER', 'file_upload_enable': upload, 'file_upload_setting': {'maxFiles': 1, 'fileLimit': 20, 'document': True, 'image': False, 'audio': False, 'video': False, 'other': False}}
|
|
previous, field = 'start-node', 'question'
|
|
if upload:
|
|
items.append(node('reader', 'component-node', '文件读取', 500, 380, fields,
|
|
{'component_id': 'file-reader', 'parameters': {'documents': ['start-node', 'document']}, 'sources': {'documents': 'reference'}, 'timeout': 60}))
|
|
previous, field = 'reader', 'text'
|
|
for index, component in enumerate(['markdown-parser', 'mindmap-engine']):
|
|
key = 'component-' + str(index)
|
|
items.append(node(key, 'component-node', registry.BUILTINS[component][0], (900 if upload else 500) + index * 420, 380, fields,
|
|
{'component_id': component, 'parameters': {'text': [previous, field]}, 'sources': {'text': 'reference'}, 'timeout': 60}))
|
|
previous, field = key, 'text'
|
|
items.append(node('reply', 'reply-node', '回复', 1750 if upload else 1340, 380, [{'label': '结果', 'value': 'answer'}],
|
|
{'reply_type': 'referencing', 'fields': [previous, 'text'], 'is_result': True}))
|
|
sequence = [item for item in items if item['id'] != 'base-node']
|
|
edges = [{'id': str(uuid.uuid4()), 'type': 'app-edge', 'sourceNodeId': left['id'], 'targetNodeId': right['id'],
|
|
'sourceAnchorId': left['id'] + '_right', 'targetAnchorId': right['id'] + '_left'} for left, right in zip(sequence, sequence[1:])]
|
|
return Application.objects.create(workspace_id=workspace, user_id=user.id, type='WORK_FLOW',
|
|
name=hub.text(data.get('name', '文档处理流程'), '流程名称', limit=64), work_flow={'nodes': items, 'edges': edges})
|