UnisKB/apps/system_manage/ai_hub_worker.py

194 lines
11 KiB
Python

"""Database-backed single consumer; never consumes legacy Celery queues."""
from django.db import transaction
from django.utils import timezone
from system_manage.models.ai_hub import AITask
from system_manage import ai_hub as hub
def claim():
from system_manage.mcp_scheduler import admit
return admit('worker')
def execute(task):
from users.models import User
if task.snapshot.get('engine'):
try:
from system_manage.mcp_workflows import execute as execute_workflow
return execute_workflow(task)
except Exception:
from system_manage.mcp_scheduler import finish
finish(task, 'FAILED', error='流程执行失败,请检查组件、授权及流程版本')
task.refresh_from_db()
return task
steps = []
try:
user = User.objects.get(pk=task.owner_id)
if not user.is_active:
raise ValueError('用户已停用')
from system_manage.platform_access import module_list
if 'mindmap' not in module_list(user):
raise ValueError('未授权使用思维导图')
if task.service_id:
service = hub.one('services', task.service_id, task.workspace_id, user)
hub.require_use(user, service)
if not service.enabled:
raise ValueError('服务已停用')
from system_manage.mcp_registry import pipeline_resources, resolve
for key in pipeline_resources(task.snapshot['nodes']): resolve(task.workspace_id, user, key, check=False)
markdown = task.input_data.get('markdown', '')
original = task.input_data.get('original', markdown)
source = task.input_data.get('heading_source', {})
if task.input_data.get('heading_structure'):
source = {**source, 'heading_structure': task.input_data['heading_structure']}
tree = {}
previous_state = task.result.get('summary_state')
previous_parsed = task.result.get('parse_checkpoint')
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree,
**({'summary_state': previous_state} if previous_state else {}),
**({'parse_checkpoint': previous_parsed} if previous_parsed else {})}
task.save(update_fields=['result', 'updated_at'])
for node in task.snapshot['nodes']:
step = {'type': node['type'], 'status': 'RUNNING'}
steps.append(step)
task.steps = steps
task.save(update_fields=['steps', 'updated_at'])
if node['type'] == 'parse' and previous_parsed and previous_parsed.get('node') == node:
markdown, original, source = (previous_parsed[key] for key in ('markdown', 'original', 'source'))
step.update(status='SUCCESS', progress={'stage': '复用已完成解析', 'percent': 100})
task.result.update(markdown=markdown, original=original, source=source)
task.save(update_fields=['steps', 'result', 'updated_at'])
continue
if node['type'] == 'parse':
parser = None
if node.get('processor') != 'builtin':
parser = hub.one('parsers', node['resource_id'], task.workspace_id, user)
hub.require_use(user, parser)
if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']:
raise ValueError('解析服务已停用或版本发生变化,请重新提交')
def report_progress(progress):
step['progress'] = progress
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(
steps=steps, updated_at=timezone.now())
report_progress({'stage': '上传文档到解析服务', 'percent': None})
from system_manage import mcp_runtime
key = 'file-reader' if parser is None else 'parser:' + str(parser.id)
parsed = mcp_runtime.invoke(task.workspace_id, user, key, {'options': node.get('options', {})},
parent=task.id, progress=report_progress, scope={'source': 'mindmap'})
markdown = parsed['text']
original = parsed.get('original', markdown)
source = parsed['structured']
task.refresh_from_db(fields=['result'])
task.result.update(original=original, markdown=markdown, source=source,
parse_checkpoint={'node': node, 'markdown': markdown, 'original': original, 'source': source})
task.save(update_fields=['result', 'updated_at'])
elif node['type'] == 'organize':
if node.get('operation') == 'heading_repair':
from system_manage.heading_service import invoke_model
def heading_progress(progress):
step['progress'] = progress
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now())
organized = invoke_model(task.workspace_id, user, node, markdown, source, task.id, heading_progress)
markdown = organized['text']
source = {**source, **organized['structured']}
else:
from system_manage import mcp_runtime
model = hub.model_for(task.workspace_id, user, node['model_id'])
if model.update_time.isoformat() != node['resource_version']:
raise ValueError('模型配置已变更,请重新提交')
def organize_progress(progress):
step['progress'] = progress
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now())
def checkpoint(outputs):
task.refresh_from_db(fields=['result'])
task.result['summary_state' if isinstance(outputs, dict) else 'organize_chunks'] = outputs
if isinstance(outputs, dict) and outputs.get('metrics'):
step.setdefault('progress', {})['plan'] = outputs['metrics']
task.save(update_fields=['result', 'steps', 'updated_at'])
params = {'text': markdown, 'instruction': node['prompt'],
'operation': node.get('operation'),
'heading_structure': source.get('heading_structure', {}), 'heading_source': source,
'resume': task.result.get('summary_state'),
**{key: node[key] for key in (
'temperature', 'top_p', 'context_tokens', 'max_input_tokens', 'max_output_tokens',
'safety_tokens', 'retries', 'detail', 'structure_mode', 'parallel', 'auto_split',
'max_depth', 'max_children', 'node_max_chars', 'concurrency', 'input_tokens') if key in node}}
if node.get('output_tokens') is not None: params['max_tokens'] = node['output_tokens']
organized = mcp_runtime.invoke(task.workspace_id, user, 'model:' + str(model.id), params,
parent=task.id, timeout=node.get('timeout', 180), progress=organize_progress,
checkpoint=checkpoint, scope={'source': 'mindmap'})
markdown = organized['text']
if isinstance(organized.get('structured'), dict):
source = {**source, **organized['structured']}
else:
from system_manage import mcp_runtime
tree = mcp_runtime.invoke(task.workspace_id, user, 'mindmap-engine', {'text': markdown,
'heading_structure': source.get('heading_structure')},
parent=task.id, scope={'source': 'mindmap'})['structured']
step['status'] = 'SUCCESS'
task.steps = steps
# Persist intermediate original output so a later model failure does not lose OCR work.
task.refresh_from_db(fields=['result'])
task.result.update(original=original, markdown=markdown, source=source, tree=tree)
task.save(update_fields=['steps', 'result', 'updated_at'])
if task.snapshot.get('workbench_artifact') and not markdown.strip():
raise ValueError('处理结果为空,请检查文档内容后重试')
task.status = 'SUCCESS'
except Exception as exc:
if steps:
steps[-1]['status'] = 'FAILED'
# Do not echo URLs, credentials, request bodies or provider exceptions to users/logs.
from rest_framework.exceptions import APIException
if isinstance(exc, APIException):
detail = exc.detail
while isinstance(detail, (list, tuple, dict)) and detail:
detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0]
task.error = str(detail)[:512]
else:
task.error = str(exc)[:512] if isinstance(exc, ValueError) else '执行失败,请检查服务状态后重试'
task.status = 'FAILED'
task.steps = steps
task.finished_at = timezone.now()
with transaction.atomic():
task.save(update_fields=['status', 'error', 'steps', 'finished_at', 'updated_at'])
if task.status == 'SUCCESS':
persist_workbench_result(task)
return task
def persist_workbench_result(task):
"""Publish only to the unchanged artifact that still points at this task."""
from system_manage.models.ai_hub import AIArtifact
from system_manage.heading_structure import store_artifact_state, store_summary_state
spec = task.snapshot.get('workbench_artifact') or {}
if not spec.get('id'):
return
artifact = AIArtifact.objects.select_for_update().filter(
pk=spec['id'], workspace_id=task.workspace_id, owner_id=task.owner_id,
task_id=task.id).first()
if not artifact:
return
if artifact.revision != spec.get('revision'):
meta = next((x.copy() for x in reversed(artifact.history or []) if isinstance(x, dict)
and x.get('_type') == 'workbench_tasks_v1'), {'_type': 'workbench_tasks_v1'})
meta['conflict_task_id'] = str(task.id)
artifact.history = [x for x in (artifact.history or []) if not isinstance(x, dict)
or x.get('_type') != 'workbench_tasks_v1'] + [meta]
artifact.save(update_fields=['history', 'updated_at'])
return
result = task.result or {}
markdown = result.get('markdown', '')
if not markdown:
return
is_summary = any(n.get('type') == 'organize' for n in task.snapshot.get('nodes', []))
if is_summary:
store_summary_state(artifact, {'source_markdown': task.input_data.get('markdown', ''),
'smart_markdown': markdown, 'mode': 'smart'})
artifact.original = result.get('original', artifact.original)
artifact.markdown = markdown
structure = (result.get('source') or {}).get('heading_structure')
if structure:
store_artifact_state(artifact, structure)
artifact.revision += 1
artifact.save(update_fields=['original', 'markdown', 'history', 'revision', 'updated_at'])