674 lines
36 KiB
Python
674 lines
36 KiB
Python
"""AI hub business operations. Secrets never cross the browser boundary."""
|
||
from urllib.parse import urlsplit
|
||
from django.db import connection, transaction
|
||
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
|
||
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
|
||
|
||
MAX_TEXT = 2_000_000
|
||
MAX_INLINE_TEXT = MAX_TEXT + 140_000_000
|
||
MAX_FILE = 20 * 1024 * 1024
|
||
KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService,
|
||
'tasks': AITask, 'artifacts': AIArtifact}
|
||
|
||
|
||
def admin(user):
|
||
from system_manage.platform_access import is_admin
|
||
return is_admin(user)
|
||
|
||
|
||
def require_admin(user):
|
||
if not admin(user):
|
||
raise PermissionDenied('仅管理员可操作')
|
||
|
||
|
||
def visible(query, user):
|
||
# Sharing is explicit, with no implicit public access.
|
||
if admin(user):
|
||
return query
|
||
if query.model in (AIParserResource, AIService):
|
||
from system_manage.platform_access import grants_for, expand_actions
|
||
kind = 'PARSER' if query.model == AIParserResource else 'SERVICE'
|
||
granted = {g.target for g in grants_for(user).filter(resource_type=kind) if 'VIEW' in expand_actions(g.actions)}
|
||
ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id)
|
||
or str(user.id) in obj.allowed_users or str(obj.id) in granted]
|
||
return query.filter(id__in=ids)
|
||
return query.filter(owner_id=user.id)
|
||
|
||
|
||
def one(kind, pk, workspace, user, edit=False, lock=False):
|
||
from rest_framework.serializers import UUIDField
|
||
pk = UUIDField().run_validation(pk)
|
||
model = KINDS.get(kind)
|
||
if not model:
|
||
raise NotFound()
|
||
query = model.objects.filter(workspace_id=workspace, id=pk)
|
||
if lock:
|
||
query = query.select_for_update()
|
||
obj = visible(query, user).first()
|
||
if not obj:
|
||
raise NotFound('记录不存在或无权访问')
|
||
if edit and not admin(user) and str(obj.owner_id) != str(user.id):
|
||
from system_manage.platform_access import resource_actions
|
||
resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind)
|
||
if not resource_type or 'EDIT' not in resource_actions(user, resource_type, obj.id, workspace):
|
||
raise PermissionDenied('无权修改')
|
||
return obj
|
||
|
||
|
||
def require_use(user, obj):
|
||
if admin(user) or str(obj.owner_id) == str(user.id) or str(user.id) in obj.allowed_users:
|
||
return
|
||
from system_manage.platform_access import resource_actions
|
||
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
|
||
if 'USE' not in resource_actions(user, kind, obj.id, obj.workspace_id):
|
||
raise PermissionDenied('未授权使用该资源')
|
||
|
||
|
||
def require_resource_action(user, obj, action):
|
||
if admin(user) or str(obj.owner_id) == str(user.id):
|
||
return
|
||
from system_manage.platform_access import resource_actions
|
||
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
|
||
if action not in resource_actions(user, kind, obj.id, obj.workspace_id):
|
||
raise PermissionDenied('无权操作该资源')
|
||
|
||
|
||
def text(value, label, required=True, limit=MAX_TEXT):
|
||
if not isinstance(value, str) or len(value) > limit or (required and not value.strip()):
|
||
raise ValidationError(f'{label}无效')
|
||
return value
|
||
|
||
|
||
def integer(value, minimum, maximum, label):
|
||
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
|
||
raise ValidationError(f'{label}无效')
|
||
return value
|
||
|
||
|
||
def users(value):
|
||
from users.models import User
|
||
if not isinstance(value, list) or len(value) > 500:
|
||
raise ValidationError('授权用户无效')
|
||
values = {str(item) for item in value}
|
||
try:
|
||
allowed = set(str(pk) for pk in User.objects.filter(id__in=values, is_active=True).values_list('id', flat=True))
|
||
except (ValueError, TypeError):
|
||
raise ValidationError('授权用户无效')
|
||
# Registration approval is enforced by the existing activation extension.
|
||
from users.models.user_manage_ext import UserActivationApplication
|
||
pending = set(str(pk) for pk in UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True))
|
||
if values - (allowed - pending):
|
||
raise ValidationError('只能授权已激活且启用的用户')
|
||
return sorted(values)
|
||
|
||
|
||
def authorized_models(workspace, user):
|
||
from models_provider.models import Model
|
||
from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType
|
||
query = Model.objects.filter(workspace_id=workspace, status='SUCCESS')
|
||
if not admin(user):
|
||
grants = WorkspaceUserResourcePermission.objects.filter(
|
||
workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL)
|
||
ids = [g.target for g in grants if 'VIEW' in g.permission_list]
|
||
from system_manage.platform_access import grants_for, expand_actions
|
||
ids.extend(g.target for g in grants_for(user).filter(resource_type='MODEL') if 'USE' in expand_actions(g.actions))
|
||
query = query.filter(id__in=ids)
|
||
return query
|
||
|
||
|
||
def model_for(workspace, user, model_id):
|
||
from rest_framework.serializers import UUIDField
|
||
model_id = UUIDField().run_validation(model_id)
|
||
obj = authorized_models(workspace, user).filter(id=model_id, model_type='LLM').first()
|
||
if not obj:
|
||
raise PermissionDenied('模型不可用或未授权')
|
||
return obj
|
||
|
||
|
||
def serialize(obj, detail=False, user=None):
|
||
data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()}
|
||
if user is not None:
|
||
data['can_delete'] = admin(user) or str(obj.owner_id) == str(user.id)
|
||
if user is not None and isinstance(obj, (AIParserResource, AIService)):
|
||
from system_manage.platform_access import resource_actions, ACTIONS
|
||
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
|
||
actions = set(ACTIONS) if admin(user) or str(obj.owner_id) == str(user.id) else resource_actions(user, kind, obj.id, obj.workspace_id)
|
||
if str(user.id) in obj.allowed_users:
|
||
actions |= {'VIEW', 'USE'}
|
||
data['actions'] = sorted(actions)
|
||
data['can_delete'] = 'EDIT' in actions
|
||
if isinstance(obj, AIParserResource):
|
||
data.update(enabled=obj.enabled, allowed_users=obj.allowed_users,
|
||
options=obj.options, credential_configured=bool(obj.credential))
|
||
# Only callers of admin management endpoints receive endpoint addresses.
|
||
elif isinstance(obj, AIFlow):
|
||
data.update(revision=obj.revision, nodes=obj.nodes)
|
||
elif isinstance(obj, AIService):
|
||
from system_manage.ai_hub_service_api import get_config
|
||
data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users,
|
||
requests_per_minute=obj.requests_per_minute,
|
||
key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run', api_config=get_config(obj.id))
|
||
elif isinstance(obj, AITask):
|
||
from system_manage.ai_hub_progress import task_progress
|
||
data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt,
|
||
started_at=obj.started_at, finished_at=obj.finished_at)
|
||
if 'snapshot' not in obj.get_deferred_fields():
|
||
data['artifact_id'] = (obj.snapshot.get('workbench_artifact') or {}).get('id')
|
||
if detail:
|
||
data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj))
|
||
if any(n.get('type') == 'organize' for n in obj.snapshot.get('nodes', [])):
|
||
data['summary_input'] = {key: obj.input_data[key] for key in ('markdown', 'heading_structure') if key in obj.input_data}
|
||
elif isinstance(obj, AIArtifact):
|
||
data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None,
|
||
task_status=obj.task.status if obj.task_id else None, task_error=obj.task.error if obj.task_id else '')
|
||
meta = next((x for x in reversed(obj.history or []) if isinstance(x, dict) and x.get('_type') == 'workbench_tasks_v1'), {})
|
||
task_ids = [meta.get('parse_task_id'), meta.get('summary_task_id')]
|
||
states = {str(pk): status for pk, status in AITask.objects.filter(
|
||
id__in=[x for x in task_ids if x]).values_list('id', 'status')}
|
||
conflict = bool(obj.task_id and meta.get('conflict_task_id') == str(obj.task_id))
|
||
if conflict:
|
||
states[str(obj.task_id)] = 'CONFLICT'
|
||
data.update(file_name=meta.get('file_name', ''), parse_status=states.get(task_ids[0]),
|
||
summary_status=states.get(task_ids[1]), task_result_conflict=conflict)
|
||
if detail:
|
||
from system_manage.heading_structure import artifact_state, version_history, summary_state
|
||
data.update(original=obj.original, markdown=obj.markdown, history=version_history(obj), heading_structure=artifact_state(obj), summary_state=summary_state(obj))
|
||
return data
|
||
|
||
|
||
def parser_save(workspace, user, data, obj=None):
|
||
if obj is None:
|
||
require_admin(user)
|
||
else:
|
||
if set(data) - {'allowed_users'}:
|
||
require_resource_action(user, obj, 'EDIT')
|
||
if 'allowed_users' in data:
|
||
require_resource_action(user, obj, 'AUTHORIZE')
|
||
endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/')
|
||
parsed = urlsplit(endpoint)
|
||
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||
raise ValidationError('解析服务地址无效')
|
||
obj = obj or AIParserResource(workspace_id=workspace, owner_id=user.id)
|
||
obj.name = text(data.get('name', obj.name), '名称', limit=128)
|
||
obj.endpoint = endpoint
|
||
if 'api_key' in data:
|
||
from common.utils.rsa_util import rsa_long_encrypt
|
||
key = text(data['api_key'], '密钥', required=False, limit=4096)
|
||
obj.credential = rsa_long_encrypt(key) if key else ''
|
||
enabled = data.get('enabled', obj.enabled)
|
||
if not isinstance(enabled, bool):
|
||
raise ValidationError('启用状态无效')
|
||
obj.enabled = enabled
|
||
if 'allowed_users' in data:
|
||
obj.allowed_users = users(data['allowed_users'])
|
||
options = data.get('options', obj.options)
|
||
from system_manage.parsers.catalog import validate_options
|
||
obj.options = validate_options(options, endpoint)
|
||
obj.save()
|
||
return obj
|
||
|
||
|
||
def parser_test(parser):
|
||
"""Probe the registered service API, without submitting files or returning service content."""
|
||
import time
|
||
import requests
|
||
from urllib.parse import urlunsplit
|
||
from common.utils.rsa_util import rsa_long_decrypt
|
||
parsed = urlsplit(parser.endpoint.rstrip('/'))
|
||
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password:
|
||
raise ValidationError('解析服务地址无效')
|
||
base_path = parsed.path
|
||
if parser.options.get('protocol') == 'v1':
|
||
if base_path.endswith('/v1'):
|
||
base_path = base_path[:-3]
|
||
url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', ''))
|
||
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
|
||
started = time.monotonic()
|
||
try:
|
||
response = requests.get(url, headers=headers, timeout=(3, 7), allow_redirects=False)
|
||
if response.status_code != 200:
|
||
raise ValidationError(f'连接测试失败(HTTP {response.status_code})')
|
||
data = response.json()
|
||
except requests.RequestException as exc:
|
||
raise ValidationError('解析服务连接失败或超时') from exc
|
||
except ValueError as exc:
|
||
raise ValidationError('解析服务未返回有效接口定义') from exc
|
||
paths = data.get('paths') if isinstance(data, dict) else None
|
||
v1 = parser.options.get('protocol') == 'v1'
|
||
required = ('/v1/uploads', '/v1/parse/jobs') if v1 else ('/file_parse',)
|
||
if not isinstance(paths, dict) or not all(any(path.endswith(suffix) and 'post' in methods
|
||
for path, methods in paths.items() if isinstance(methods, dict)) for suffix in required):
|
||
raise ValidationError('服务接口与所选 MinerU 版本不匹配')
|
||
# Existence of the API does not prove that a local GPU/model backend is ready.
|
||
return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000),
|
||
'api_version': data.get('info', {}).get('version', ''), 'message': '解析接口连接正常'}
|
||
|
||
|
||
|
||
def validate_nodes(workspace, user, nodes):
|
||
if not isinstance(nodes, list) or not 1 <= len(nodes) <= 3:
|
||
raise ValidationError('请选择处理节点')
|
||
types = [n.get('type') for n in nodes if isinstance(n, dict)]
|
||
if len(types) != len(nodes) or types[-1] != 'render' or len(set(types)) != len(types):
|
||
raise ValidationError('流程须以导图生成为终点,节点不可重复')
|
||
if types != [t for t in ('parse', 'organize', 'render') if t in types]:
|
||
raise ValidationError('节点顺序应为解析、整理、导图生成')
|
||
result = []
|
||
for node in nodes:
|
||
kind = node['type']
|
||
clean = {'type': kind}
|
||
if kind == 'parse':
|
||
if node.get('processor') == 'builtin':
|
||
result.append({'type': 'parse', 'processor': 'builtin'})
|
||
continue
|
||
parser = one('parsers', node.get('resource_id'), workspace, user, lock=connection.in_atomic_block)
|
||
require_use(user, parser)
|
||
if not parser.enabled:
|
||
raise ValidationError('解析服务已停用')
|
||
clean['resource_id'] = str(parser.id)
|
||
if kind == 'organize':
|
||
if node.get('operation') not in (None, 'heading_repair'):
|
||
raise ValidationError('整理操作无效')
|
||
if node.get('operation'):
|
||
clean['operation'] = node['operation']
|
||
clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id)
|
||
clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000)
|
||
from system_manage.summary_plan import validate_options
|
||
from system_manage.ai_hub_organize import resolve_settings
|
||
from models_provider.tools import get_model_default_params
|
||
from system_manage.mcp_business import summary_model_defaults, summary_supported_parameters
|
||
try:
|
||
clean.update(validate_options(node))
|
||
model = model_for(workspace, user, clean['model_id'])
|
||
resolve_settings(summary_model_defaults(model, get_model_default_params(model)), clean)
|
||
if clean.get('top_p') is not None and 'top_p' not in summary_supported_parameters(model):
|
||
raise ValueError('该模型未声明支持 top_p 参数')
|
||
except ValueError as exc:
|
||
raise ValidationError(str(exc)) from None
|
||
result.append(clean)
|
||
return result
|
||
|
||
|
||
@transaction.atomic
|
||
def flow_create(workspace, user, data):
|
||
nodes = validate_nodes(workspace, user, data.get('nodes'))
|
||
# Every save is immutable; published services always reference a fixed row.
|
||
previous = one('flows', data['previous_id'], workspace, user, edit=True, lock=True) if data.get('previous_id') else None
|
||
return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id,
|
||
name=text(data.get('name'), '名称', limit=128), nodes=nodes,
|
||
revision=previous.revision + 1 if previous else 1)
|
||
|
||
|
||
@transaction.atomic
|
||
def service_create(workspace, user, data):
|
||
require_admin(user)
|
||
flow = one('flows', data.get('flow_id'), workspace, user, lock=True)
|
||
validate_nodes(workspace, user, flow.nodes)
|
||
obj = AIService.objects.create(workspace_id=workspace, owner_id=user.id,
|
||
name=text(data.get('name'), '名称', limit=128), flow=flow,
|
||
allowed_users=users(data.get('allowed_users', [])),
|
||
requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率'))
|
||
if 'api_config' in data:
|
||
from system_manage.ai_hub_service_api import save_config
|
||
save_config(obj.id, data['api_config'])
|
||
return obj
|
||
|
||
|
||
def make_snapshot(workspace, user, nodes):
|
||
clean = validate_nodes(workspace, user, nodes)
|
||
for node in clean:
|
||
if node['type'] == 'parse':
|
||
if node.get('processor') == 'builtin':
|
||
continue
|
||
parser = one('parsers', node['resource_id'], workspace, user)
|
||
node['options'] = dict(parser.options)
|
||
node['resource_version'] = parser.updated_at.isoformat()
|
||
if node['type'] == 'organize':
|
||
model = model_for(workspace, user, node['model_id'])
|
||
node.update(model_name=model.model_name, provider=model.provider, resource_version=model.update_time.isoformat())
|
||
return clean
|
||
|
||
|
||
@transaction.atomic
|
||
def submit(workspace, user, data, upload=None, service=None, request=None):
|
||
from users.models import User
|
||
current_user = User.objects.select_for_update().get(id=user.id)
|
||
if not current_user.is_active:
|
||
raise PermissionDenied('用户已停用')
|
||
if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
|
||
raise ValidationError('待处理任务已达上限')
|
||
if service:
|
||
service = one('services', service.pk, workspace, user, lock=True)
|
||
require_use(user, service)
|
||
if not service.enabled:
|
||
raise ValidationError('服务已停用')
|
||
nodes = service.flow.nodes
|
||
else:
|
||
nodes = one('flows', data['flow_id'], workspace, user, lock=True).nodes if data.get('flow_id') else data.get('nodes')
|
||
snapshot = make_snapshot(workspace, user, nodes)
|
||
from system_manage.mcp_registry import pipeline_resources, resolve
|
||
required = pipeline_resources(snapshot)
|
||
for key in required: resolve(workspace, user, key)
|
||
priority = integer(data.get('priority', 5), 0, 9, '优先级')
|
||
from system_manage.access_control import enforce_task
|
||
enforce_task(request, service.id if service else None, snapshot)
|
||
markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT)
|
||
if upload and upload.size > MAX_FILE:
|
||
raise ValidationError('文件不能超过20MB')
|
||
if upload and snapshot[0]['type'] != 'parse':
|
||
raise ValidationError('文件任务需要解析节点')
|
||
if not upload and snapshot[0]['type'] == 'parse':
|
||
raise ValidationError('请选择文件')
|
||
if not upload and not markdown.strip():
|
||
raise ValidationError('文档内容不能为空')
|
||
from system_manage.ai_hub_assets import asset_lock, normalize
|
||
with asset_lock():
|
||
markdown = text(normalize(markdown, workspace, user), '文档内容', required=False)
|
||
heading_input = {}
|
||
if data.get('heading_structure') is not None:
|
||
from system_manage.heading_structure import analyze, validate_manual
|
||
if not isinstance(data['heading_structure'], dict):
|
||
raise ValidationError('层级数据无效')
|
||
try:
|
||
_, structure = analyze(markdown, previous=data['heading_structure'], repair=False)
|
||
structure = validate_manual(structure, data['heading_structure'])
|
||
except ValueError as exc:
|
||
raise ValidationError(str(exc)) from None
|
||
heading_input = {'heading_structure': structure, 'original': text(data.get('original', markdown), '原稿', required=False)}
|
||
if data.get('_heading_source') is not None:
|
||
heading_input['heading_source'] = data['_heading_source']
|
||
# Plan without invoking a model. Reserve only useful internal parallel calls.
|
||
from system_manage.mcp_registry import settings_row
|
||
from system_manage.mcp_business import summary_model_defaults, summary_counter
|
||
from system_manage.summary_plan import token_counter
|
||
from system_manage.ai_hub_organize import plan_document, resolve_settings
|
||
from models_provider.tools import get_model_default_params
|
||
from system_manage.ai_hub_assets import protect_images
|
||
scheduler_settings = settings_row().meta
|
||
capacity = scheduler_settings.get('capacity', 4)
|
||
worker_capacity = max(1, capacity - min(scheduler_settings.get('interactive_reserve', 1), max(0, capacity - 1)))
|
||
resource_slots = {}
|
||
for node in snapshot:
|
||
if node['type'] != 'organize' or node.get('operation') == 'heading_repair':
|
||
continue
|
||
key = 'model:' + node['model_id']
|
||
slots = min(node.get('concurrency', 1) if node.get('parallel', True) else 1,
|
||
resolve(workspace, user, key)[0]['capacity'], worker_capacity)
|
||
if snapshot[0]['type'] != 'parse':
|
||
model = model_for(workspace, user, node['model_id'])
|
||
defaults = summary_model_defaults(model, get_model_default_params(model))
|
||
try:
|
||
options = {**node, 'max_input_tokens': min(int(defaults.get('max_input_tokens', 2000000)), node.get('max_input_tokens', 2000000))}
|
||
if 'input_tokens' not in options and defaults.get('input_tokens'):
|
||
options['input_tokens'] = int(defaults['input_tokens'])
|
||
settings = resolve_settings(defaults, node)
|
||
count, _ = summary_counter(model, defaults, markdown, workspace, user.id, settings, options)
|
||
if node.get('structure_mode', 'preserve') in ('preserve', 'reorganize'):
|
||
from system_manage.summary_sections import build_plan
|
||
planned = build_plan(protect_images(markdown)[0], node['prompt'], settings, options, count)
|
||
slots = min(slots, max(1, len(planned['batches'])))
|
||
else:
|
||
_, _, _, _, chunks = plan_document(protect_images(markdown)[0], node['prompt'], settings, options, count)
|
||
slots = min(slots, len(chunks))
|
||
except ValueError as exc:
|
||
raise ValidationError(str(exc)) from None
|
||
resource_slots[key] = slots
|
||
task = AITask.objects.create(workspace_id=workspace, owner_id=user.id,
|
||
name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service,
|
||
snapshot={'nodes': snapshot, 'resources': required, 'resource_slots': resource_slots, 'priority': priority, 'flow_id': str(service.flow_id) if service else data.get('flow_id'),
|
||
'flow_revision': service.flow.revision if service else None},
|
||
input_data={'markdown': markdown, **heading_input}, file_name=upload.name[:255] if upload else '',
|
||
file_data=upload.read() if upload else None)
|
||
if 'workbench_artifact' in data:
|
||
spec = data['workbench_artifact']
|
||
if not isinstance(spec, dict) or set(spec) - {'id', 'revision', 'layout'}:
|
||
raise ValidationError('导图任务参数无效')
|
||
if spec.get('id'):
|
||
from rest_framework.serializers import UUIDField
|
||
artifact_id = UUIDField().run_validation(spec['id'])
|
||
artifact = AIArtifact.objects.select_for_update().filter(
|
||
pk=artifact_id, workspace_id=workspace, owner_id=user.id).first()
|
||
if not artifact:
|
||
raise ValidationError('导图不存在或无权访问')
|
||
if type(spec.get('revision')) is not int or spec['revision'] != artifact.revision:
|
||
raise ValidationError('内容已更新,请重新打开')
|
||
else:
|
||
from users.models import User
|
||
from system_manage.artifact_settings import limit
|
||
User.objects.select_for_update().get(pk=user.id)
|
||
maximum = limit()
|
||
if AIArtifact.objects.filter(workspace_id=workspace, owner_id=user.id).count() >= maximum:
|
||
raise ValidationError(f'最多可保存 {maximum} 张导图,请先删除不需要的导图')
|
||
layout = spec.get('layout', 'tree')
|
||
if layout not in ('tree', 'radial', 'horizontal', 'vertical'):
|
||
raise ValidationError('布局无效')
|
||
artifact = AIArtifact.objects.create(workspace_id=workspace, owner_id=user.id,
|
||
name=task.name, layout=layout, original=heading_input.get('original', markdown), markdown=markdown)
|
||
artifact.task = task
|
||
meta = next((x.copy() for x in reversed(artifact.history or []) if isinstance(x, dict) and x.get('_type') == 'workbench_tasks_v1'), {'_type': 'workbench_tasks_v1'})
|
||
meta.pop('conflict_task_id', None)
|
||
is_summary = any(node['type'] == 'organize' for node in snapshot)
|
||
meta['summary_task_id' if is_summary else 'parse_task_id'] = str(task.id)
|
||
if upload:
|
||
meta['file_name'] = upload.name[:255]
|
||
artifact.history = [x for x in (artifact.history or []) if not isinstance(x, dict) or x.get('_type') != 'workbench_tasks_v1'] + [meta]
|
||
artifact.save(update_fields=['task', 'history', 'updated_at'])
|
||
task.snapshot['workbench_artifact'] = {'id': str(artifact.id), 'revision': artifact.revision}
|
||
task.save(update_fields=['snapshot', 'updated_at'])
|
||
return task
|
||
|
||
|
||
@transaction.atomic
|
||
def artifact_rename(obj, data):
|
||
if set(data) != {'action', 'name', 'revision'}:
|
||
raise ValidationError('重命名参数无效')
|
||
if isinstance(data['revision'], bool) or not isinstance(data['revision'], int) or data['revision'] != obj.revision:
|
||
raise ValidationError('内容已更新,请重新打开')
|
||
obj.name = text(data['name'], '名称', limit=128)
|
||
obj.save(update_fields=['name', 'updated_at'])
|
||
return obj
|
||
|
||
|
||
@transaction.atomic
|
||
def artifact_save(workspace, user, data, obj=None):
|
||
from system_manage.heading_structure import analyze, artifact_state, store_artifact_state, store_summary_state, summary_state, validate_manual
|
||
previous_structure = artifact_state(obj) if obj else {}
|
||
if obj:
|
||
obj = AIArtifact.objects.select_for_update().get(pk=obj.pk)
|
||
previous_structure = artifact_state(obj)
|
||
if data.get('revision') != obj.revision:
|
||
raise ValidationError('内容已更新,请重新打开')
|
||
obj.revision += 1
|
||
else:
|
||
from users.models import User
|
||
from system_manage.artifact_settings import limit
|
||
User.objects.select_for_update().get(pk=user.id)
|
||
maximum = limit()
|
||
if AIArtifact.objects.filter(workspace_id=workspace, owner_id=user.id).count() >= maximum:
|
||
raise ValidationError(f'最多可保存 {maximum} 张导图,请先删除不需要的导图')
|
||
task = one('tasks', data['task_id'], workspace, user, lock=True) if data.get('task_id') else None
|
||
if task and task.status != 'SUCCESS':
|
||
raise ValidationError('任务尚未成功')
|
||
obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task,
|
||
original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT))
|
||
if task:
|
||
previous_structure = (task.result.get('source') or {}).get('heading_structure', {})
|
||
obj.name = text(data.get('name'), '名称', limit=128)
|
||
obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT)
|
||
obj.layout = data.get('layout', 'tree')
|
||
if obj.layout not in ('tree', 'radial', 'horizontal', 'vertical'):
|
||
raise ValidationError('布局无效')
|
||
from system_manage.ai_hub_assets import asset_lock, normalize
|
||
with asset_lock():
|
||
obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容')
|
||
obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False)
|
||
if 'summary_state' in data:
|
||
state = data['summary_state']
|
||
if not isinstance(state, dict) or set(state) - {'source_markdown', 'smart_markdown', 'mode'}:
|
||
raise ValidationError('智能总结内容无效')
|
||
if state.get('mode', 'source') not in ('source', 'smart'):
|
||
raise ValidationError('智能总结状态无效')
|
||
clean_state = {'mode': state.get('mode', 'source')}
|
||
for key in ('source_markdown', 'smart_markdown'):
|
||
clean_state[key] = text(normalize(text(state.get(key, ''), '总结内容', required=False, limit=MAX_INLINE_TEXT), workspace, user), '总结内容', required=False, limit=MAX_INLINE_TEXT)
|
||
store_summary_state(obj, clean_state)
|
||
elif summary_state(obj):
|
||
state = dict(summary_state(obj))
|
||
state.pop('_type', None)
|
||
state['smart_markdown' if state.get('mode') == 'smart' else 'source_markdown'] = obj.markdown
|
||
store_summary_state(obj, state)
|
||
for entry in obj.history:
|
||
if 'markdown' in entry:
|
||
entry['markdown'] = normalize(entry['markdown'], workspace, user)
|
||
incoming = data.get('heading_structure')
|
||
if incoming is not None or previous_structure:
|
||
try:
|
||
_, structure = analyze(obj.markdown, previous=previous_structure or incoming, repair=False)
|
||
if incoming is not None:
|
||
structure = validate_manual(structure, incoming)
|
||
_, structure = analyze(obj.markdown, previous=structure, repair=False)
|
||
except ValueError as exc:
|
||
raise ValidationError(str(exc)) from None
|
||
store_artifact_state(obj, structure)
|
||
obj.save()
|
||
return obj
|
||
|
||
|
||
@transaction.atomic
|
||
def delete_record(kind, pk, workspace, user):
|
||
obj = one(kind, pk, workspace, user, edit=True, lock=True)
|
||
if kind == 'parsers':
|
||
reference = [{'type': 'parse', 'resource_id': str(obj.id)}]
|
||
if AIFlow.objects.filter(workspace_id=workspace, nodes__contains=reference).exists():
|
||
raise ValidationError('解析服务已被流程引用,请先删除相关流程')
|
||
if AITask.objects.filter(workspace_id=workspace, status__in=['PENDING', 'RUNNING'],
|
||
snapshot__nodes__contains=reference).exists():
|
||
raise ValidationError('解析服务有未完成任务,请稍后删除')
|
||
elif kind == 'flows':
|
||
if AIService.objects.filter(flow=obj).exists():
|
||
raise ValidationError('流程已被服务引用,请先删除相关服务')
|
||
elif kind == 'services':
|
||
if AITask.objects.filter(service=obj, status__in=['PENDING', 'RUNNING']).exists():
|
||
raise ValidationError('服务有未完成任务,请稍后删除')
|
||
elif kind == 'tasks':
|
||
if obj.status in ('PENDING', 'RUNNING'):
|
||
raise ValidationError('待处理或运行中的任务不能删除')
|
||
elif kind != 'artifacts':
|
||
raise ValidationError('资源类型无效')
|
||
# Historical task snapshots and saved artifacts remain independent records.
|
||
resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind)
|
||
if resource_type:
|
||
from system_manage.models.platform_access import PlatformGrant
|
||
PlatformGrant.objects.filter(workspace_id=workspace, resource_type=resource_type,
|
||
target=str(obj.id)).delete()
|
||
if kind == 'services':
|
||
from system_manage.ai_hub_service_api import save_config
|
||
save_config(obj.id)
|
||
obj.delete()
|
||
|
||
|
||
def parse_file(parser, name, content, options=None, progress_callback=None):
|
||
import requests
|
||
from common.utils.rsa_util import rsa_long_decrypt
|
||
opts = options if options is not None else parser.options
|
||
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
|
||
from system_manage.parsers import parse
|
||
return parse(parser, name, content, opts, headers, progress_callback=progress_callback)
|
||
|
||
|
||
def markdown_tree(markdown, excluded_lines=None):
|
||
"""Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。"""
|
||
import re
|
||
excluded_lines = set(excluded_lines or [])
|
||
root = {'id': 'root', 'text': '文档', 'children': []}
|
||
heading_stack = [(0, root)]
|
||
list_stack = []
|
||
fence = None
|
||
table_buffer, table_start = [], None
|
||
|
||
def container():
|
||
return list_stack[-1][1] if list_stack else heading_stack[-1][1]
|
||
|
||
def flush_table():
|
||
nonlocal table_buffer, table_start
|
||
if not table_buffer:
|
||
return
|
||
index, lines = table_start, table_buffer
|
||
table_buffer, table_start = [], None
|
||
|
||
def cells(raw):
|
||
return [cell.strip() for cell in raw.strip().strip('|').split('|')]
|
||
|
||
if len(lines) >= 2 and re.match(r'^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$', lines[1]):
|
||
headers = cells(lines[0])
|
||
aligns = []
|
||
for cell in cells(lines[1]):
|
||
left, right = cell.startswith(':'), cell.endswith(':')
|
||
aligns.append('center' if left and right else 'right' if right else 'left')
|
||
rows = [cells(raw) for raw in lines[2:]]
|
||
node = {'id': f'n{index}', 'type': 'table', 'text': '表格:' + '、'.join(headers),
|
||
'table': {'headers': headers, 'aligns': aligns, 'rows': rows,
|
||
'row_count': len(rows), 'column_count': len(headers)},
|
||
'children': []}
|
||
container()['children'].append(node)
|
||
else:
|
||
for offset, raw in enumerate(lines):
|
||
container()['children'].append({'id': f'n{index + offset}', 'text': raw.strip(), 'children': []})
|
||
|
||
list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$')
|
||
for index, line in enumerate(markdown.splitlines()):
|
||
if excluded_lines and index in excluded_lines:
|
||
continue
|
||
fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
|
||
if fence_match:
|
||
token = fence_match.group(1)
|
||
if fence is None:
|
||
fence = token
|
||
flush_table()
|
||
list_stack.clear()
|
||
elif token[0] == fence[0] and len(token) >= len(fence):
|
||
fence = None
|
||
continue
|
||
if fence:
|
||
continue
|
||
stripped = line.strip()
|
||
if stripped.startswith('|') and stripped.count('|') >= 2:
|
||
if table_start is None:
|
||
table_start = index
|
||
list_stack.clear()
|
||
table_buffer.append(line)
|
||
continue
|
||
flush_table()
|
||
if not stripped:
|
||
continue
|
||
heading = re.match(r'^\s{0,3}(#{1,6})\s+(.+)', line)
|
||
if heading:
|
||
level = len(heading.group(1))
|
||
list_stack.clear()
|
||
while heading_stack[-1][0] >= level:
|
||
heading_stack.pop()
|
||
node = {'id': f'n{index}', 'text': heading.group(2).rstrip('# ').strip(), 'children': []}
|
||
heading_stack[-1][1]['children'].append(node)
|
||
heading_stack.append((level, node))
|
||
continue
|
||
item = list_pattern.match(line)
|
||
if item:
|
||
indent = len(item.group(1).expandtabs(4))
|
||
while list_stack and list_stack[-1][0] >= indent:
|
||
list_stack.pop()
|
||
ordered = item.group(3)
|
||
node = {'id': f'n{index}', 'type': 'list-item',
|
||
'text': item.group(4).strip(),
|
||
'list': {'type': 'ol' if ordered else 'ul', 'index': int(ordered) if ordered else None},
|
||
'children': []}
|
||
container()['children'].append(node)
|
||
list_stack.append((indent, node))
|
||
continue
|
||
if line[:1].isspace() and list_stack:
|
||
list_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
|
||
continue
|
||
list_stack.clear()
|
||
heading_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
|
||
flush_table()
|
||
return root
|