194 lines
11 KiB
Python
194 lines
11 KiB
Python
"""Database-backed single consumer; never consumes legacy Celery queues."""
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
from system_manage.models.ai_hub import AITask
|
|
from system_manage import ai_hub as hub
|
|
|
|
|
|
def claim():
|
|
from system_manage.mcp_scheduler import admit
|
|
return admit('worker')
|
|
|
|
|
|
def execute(task):
|
|
from users.models import User
|
|
if task.snapshot.get('engine'):
|
|
try:
|
|
from system_manage.mcp_workflows import execute as execute_workflow
|
|
return execute_workflow(task)
|
|
except Exception:
|
|
from system_manage.mcp_scheduler import finish
|
|
finish(task, 'FAILED', error='流程执行失败,请检查组件、授权及流程版本')
|
|
task.refresh_from_db()
|
|
return task
|
|
steps = []
|
|
try:
|
|
user = User.objects.get(pk=task.owner_id)
|
|
if not user.is_active:
|
|
raise ValueError('用户已停用')
|
|
from system_manage.platform_access import module_list
|
|
if 'mindmap' not in module_list(user):
|
|
raise ValueError('未授权使用思维导图')
|
|
if task.service_id:
|
|
service = hub.one('services', task.service_id, task.workspace_id, user)
|
|
hub.require_use(user, service)
|
|
if not service.enabled:
|
|
raise ValueError('服务已停用')
|
|
from system_manage.mcp_registry import pipeline_resources, resolve
|
|
for key in pipeline_resources(task.snapshot['nodes']): resolve(task.workspace_id, user, key, check=False)
|
|
markdown = task.input_data.get('markdown', '')
|
|
original = task.input_data.get('original', markdown)
|
|
source = task.input_data.get('heading_source', {})
|
|
if task.input_data.get('heading_structure'):
|
|
source = {**source, 'heading_structure': task.input_data['heading_structure']}
|
|
tree = {}
|
|
previous_state = task.result.get('summary_state')
|
|
previous_parsed = task.result.get('parse_checkpoint')
|
|
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree,
|
|
**({'summary_state': previous_state} if previous_state else {}),
|
|
**({'parse_checkpoint': previous_parsed} if previous_parsed else {})}
|
|
task.save(update_fields=['result', 'updated_at'])
|
|
for node in task.snapshot['nodes']:
|
|
step = {'type': node['type'], 'status': 'RUNNING'}
|
|
steps.append(step)
|
|
task.steps = steps
|
|
task.save(update_fields=['steps', 'updated_at'])
|
|
if node['type'] == 'parse' and previous_parsed and previous_parsed.get('node') == node:
|
|
markdown, original, source = (previous_parsed[key] for key in ('markdown', 'original', 'source'))
|
|
step.update(status='SUCCESS', progress={'stage': '复用已完成解析', 'percent': 100})
|
|
task.result.update(markdown=markdown, original=original, source=source)
|
|
task.save(update_fields=['steps', 'result', 'updated_at'])
|
|
continue
|
|
if node['type'] == 'parse':
|
|
parser = None
|
|
if node.get('processor') != 'builtin':
|
|
parser = hub.one('parsers', node['resource_id'], task.workspace_id, user)
|
|
hub.require_use(user, parser)
|
|
if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']:
|
|
raise ValueError('解析服务已停用或版本发生变化,请重新提交')
|
|
def report_progress(progress):
|
|
step['progress'] = progress
|
|
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(
|
|
steps=steps, updated_at=timezone.now())
|
|
report_progress({'stage': '上传文档到解析服务', 'percent': None})
|
|
from system_manage import mcp_runtime
|
|
key = 'file-reader' if parser is None else 'parser:' + str(parser.id)
|
|
parsed = mcp_runtime.invoke(task.workspace_id, user, key, {'options': node.get('options', {})},
|
|
parent=task.id, progress=report_progress, scope={'source': 'mindmap'})
|
|
markdown = parsed['text']
|
|
original = parsed.get('original', markdown)
|
|
source = parsed['structured']
|
|
task.refresh_from_db(fields=['result'])
|
|
task.result.update(original=original, markdown=markdown, source=source,
|
|
parse_checkpoint={'node': node, 'markdown': markdown, 'original': original, 'source': source})
|
|
task.save(update_fields=['result', 'updated_at'])
|
|
elif node['type'] == 'organize':
|
|
if node.get('operation') == 'heading_repair':
|
|
from system_manage.heading_service import invoke_model
|
|
def heading_progress(progress):
|
|
step['progress'] = progress
|
|
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now())
|
|
organized = invoke_model(task.workspace_id, user, node, markdown, source, task.id, heading_progress)
|
|
markdown = organized['text']
|
|
source = {**source, **organized['structured']}
|
|
else:
|
|
from system_manage import mcp_runtime
|
|
model = hub.model_for(task.workspace_id, user, node['model_id'])
|
|
if model.update_time.isoformat() != node['resource_version']:
|
|
raise ValueError('模型配置已变更,请重新提交')
|
|
def organize_progress(progress):
|
|
step['progress'] = progress
|
|
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now())
|
|
def checkpoint(outputs):
|
|
task.refresh_from_db(fields=['result'])
|
|
task.result['summary_state' if isinstance(outputs, dict) else 'organize_chunks'] = outputs
|
|
if isinstance(outputs, dict) and outputs.get('metrics'):
|
|
step.setdefault('progress', {})['plan'] = outputs['metrics']
|
|
task.save(update_fields=['result', 'steps', 'updated_at'])
|
|
params = {'text': markdown, 'instruction': node['prompt'],
|
|
'operation': node.get('operation'),
|
|
'heading_structure': source.get('heading_structure', {}), 'heading_source': source,
|
|
'resume': task.result.get('summary_state'),
|
|
**{key: node[key] for key in (
|
|
'temperature', 'top_p', 'context_tokens', 'max_input_tokens', 'max_output_tokens',
|
|
'safety_tokens', 'retries', 'detail', 'structure_mode', 'parallel', 'auto_split',
|
|
'max_depth', 'max_children', 'node_max_chars', 'concurrency', 'input_tokens') if key in node}}
|
|
if node.get('output_tokens') is not None: params['max_tokens'] = node['output_tokens']
|
|
organized = mcp_runtime.invoke(task.workspace_id, user, 'model:' + str(model.id), params,
|
|
parent=task.id, timeout=node.get('timeout', 180), progress=organize_progress,
|
|
checkpoint=checkpoint, scope={'source': 'mindmap'})
|
|
markdown = organized['text']
|
|
if isinstance(organized.get('structured'), dict):
|
|
source = {**source, **organized['structured']}
|
|
else:
|
|
from system_manage import mcp_runtime
|
|
tree = mcp_runtime.invoke(task.workspace_id, user, 'mindmap-engine', {'text': markdown,
|
|
'heading_structure': source.get('heading_structure')},
|
|
parent=task.id, scope={'source': 'mindmap'})['structured']
|
|
step['status'] = 'SUCCESS'
|
|
task.steps = steps
|
|
# Persist intermediate original output so a later model failure does not lose OCR work.
|
|
task.refresh_from_db(fields=['result'])
|
|
task.result.update(original=original, markdown=markdown, source=source, tree=tree)
|
|
task.save(update_fields=['steps', 'result', 'updated_at'])
|
|
if task.snapshot.get('workbench_artifact') and not markdown.strip():
|
|
raise ValueError('处理结果为空,请检查文档内容后重试')
|
|
task.status = 'SUCCESS'
|
|
except Exception as exc:
|
|
if steps:
|
|
steps[-1]['status'] = 'FAILED'
|
|
# Do not echo URLs, credentials, request bodies or provider exceptions to users/logs.
|
|
from rest_framework.exceptions import APIException
|
|
if isinstance(exc, APIException):
|
|
detail = exc.detail
|
|
while isinstance(detail, (list, tuple, dict)) and detail:
|
|
detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0]
|
|
task.error = str(detail)[:512]
|
|
else:
|
|
task.error = str(exc)[:512] if isinstance(exc, ValueError) else '执行失败,请检查服务状态后重试'
|
|
task.status = 'FAILED'
|
|
task.steps = steps
|
|
task.finished_at = timezone.now()
|
|
with transaction.atomic():
|
|
task.save(update_fields=['status', 'error', 'steps', 'finished_at', 'updated_at'])
|
|
if task.status == 'SUCCESS':
|
|
persist_workbench_result(task)
|
|
return task
|
|
|
|
|
|
def persist_workbench_result(task):
|
|
"""Publish only to the unchanged artifact that still points at this task."""
|
|
from system_manage.models.ai_hub import AIArtifact
|
|
from system_manage.heading_structure import store_artifact_state, store_summary_state
|
|
spec = task.snapshot.get('workbench_artifact') or {}
|
|
if not spec.get('id'):
|
|
return
|
|
artifact = AIArtifact.objects.select_for_update().filter(
|
|
pk=spec['id'], workspace_id=task.workspace_id, owner_id=task.owner_id,
|
|
task_id=task.id).first()
|
|
if not artifact:
|
|
return
|
|
if artifact.revision != spec.get('revision'):
|
|
meta = next((x.copy() for x in reversed(artifact.history or []) if isinstance(x, dict)
|
|
and x.get('_type') == 'workbench_tasks_v1'), {'_type': 'workbench_tasks_v1'})
|
|
meta['conflict_task_id'] = str(task.id)
|
|
artifact.history = [x for x in (artifact.history or []) if not isinstance(x, dict)
|
|
or x.get('_type') != 'workbench_tasks_v1'] + [meta]
|
|
artifact.save(update_fields=['history', 'updated_at'])
|
|
return
|
|
result = task.result or {}
|
|
markdown = result.get('markdown', '')
|
|
if not markdown:
|
|
return
|
|
is_summary = any(n.get('type') == 'organize' for n in task.snapshot.get('nodes', []))
|
|
if is_summary:
|
|
store_summary_state(artifact, {'source_markdown': task.input_data.get('markdown', ''),
|
|
'smart_markdown': markdown, 'mode': 'smart'})
|
|
artifact.original = result.get('original', artifact.original)
|
|
artifact.markdown = markdown
|
|
structure = (result.get('source') or {}).get('heading_structure')
|
|
if structure:
|
|
store_artifact_state(artifact, structure)
|
|
artifact.revision += 1
|
|
artifact.save(update_fields=['original', 'markdown', 'history', 'revision', 'updated_at'])
|