UnisKB/apps/system_manage/ai_hub.py

674 lines
36 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""AI hub business operations. Secrets never cross the browser boundary."""
from urllib.parse import urlsplit
from django.db import connection, transaction
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
MAX_TEXT = 2_000_000
MAX_INLINE_TEXT = MAX_TEXT + 140_000_000
MAX_FILE = 20 * 1024 * 1024
KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService,
'tasks': AITask, 'artifacts': AIArtifact}
def admin(user):
from system_manage.platform_access import is_admin
return is_admin(user)
def require_admin(user):
if not admin(user):
raise PermissionDenied('仅管理员可操作')
def visible(query, user):
# Sharing is explicit, with no implicit public access.
if admin(user):
return query
if query.model in (AIParserResource, AIService):
from system_manage.platform_access import grants_for, expand_actions
kind = 'PARSER' if query.model == AIParserResource else 'SERVICE'
granted = {g.target for g in grants_for(user).filter(resource_type=kind) if 'VIEW' in expand_actions(g.actions)}
ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id)
or str(user.id) in obj.allowed_users or str(obj.id) in granted]
return query.filter(id__in=ids)
return query.filter(owner_id=user.id)
def one(kind, pk, workspace, user, edit=False, lock=False):
from rest_framework.serializers import UUIDField
pk = UUIDField().run_validation(pk)
model = KINDS.get(kind)
if not model:
raise NotFound()
query = model.objects.filter(workspace_id=workspace, id=pk)
if lock:
query = query.select_for_update()
obj = visible(query, user).first()
if not obj:
raise NotFound('记录不存在或无权访问')
if edit and not admin(user) and str(obj.owner_id) != str(user.id):
from system_manage.platform_access import resource_actions
resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind)
if not resource_type or 'EDIT' not in resource_actions(user, resource_type, obj.id, workspace):
raise PermissionDenied('无权修改')
return obj
def require_use(user, obj):
if admin(user) or str(obj.owner_id) == str(user.id) or str(user.id) in obj.allowed_users:
return
from system_manage.platform_access import resource_actions
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
if 'USE' not in resource_actions(user, kind, obj.id, obj.workspace_id):
raise PermissionDenied('未授权使用该资源')
def require_resource_action(user, obj, action):
if admin(user) or str(obj.owner_id) == str(user.id):
return
from system_manage.platform_access import resource_actions
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
if action not in resource_actions(user, kind, obj.id, obj.workspace_id):
raise PermissionDenied('无权操作该资源')
def text(value, label, required=True, limit=MAX_TEXT):
if not isinstance(value, str) or len(value) > limit or (required and not value.strip()):
raise ValidationError(f'{label}无效')
return value
def integer(value, minimum, maximum, label):
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise ValidationError(f'{label}无效')
return value
def users(value):
from users.models import User
if not isinstance(value, list) or len(value) > 500:
raise ValidationError('授权用户无效')
values = {str(item) for item in value}
try:
allowed = set(str(pk) for pk in User.objects.filter(id__in=values, is_active=True).values_list('id', flat=True))
except (ValueError, TypeError):
raise ValidationError('授权用户无效')
# Registration approval is enforced by the existing activation extension.
from users.models.user_manage_ext import UserActivationApplication
pending = set(str(pk) for pk in UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True))
if values - (allowed - pending):
raise ValidationError('只能授权已激活且启用的用户')
return sorted(values)
def authorized_models(workspace, user):
from models_provider.models import Model
from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType
query = Model.objects.filter(workspace_id=workspace, status='SUCCESS')
if not admin(user):
grants = WorkspaceUserResourcePermission.objects.filter(
workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL)
ids = [g.target for g in grants if 'VIEW' in g.permission_list]
from system_manage.platform_access import grants_for, expand_actions
ids.extend(g.target for g in grants_for(user).filter(resource_type='MODEL') if 'USE' in expand_actions(g.actions))
query = query.filter(id__in=ids)
return query
def model_for(workspace, user, model_id):
from rest_framework.serializers import UUIDField
model_id = UUIDField().run_validation(model_id)
obj = authorized_models(workspace, user).filter(id=model_id, model_type='LLM').first()
if not obj:
raise PermissionDenied('模型不可用或未授权')
return obj
def serialize(obj, detail=False, user=None):
data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()}
if user is not None:
data['can_delete'] = admin(user) or str(obj.owner_id) == str(user.id)
if user is not None and isinstance(obj, (AIParserResource, AIService)):
from system_manage.platform_access import resource_actions, ACTIONS
kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE'
actions = set(ACTIONS) if admin(user) or str(obj.owner_id) == str(user.id) else resource_actions(user, kind, obj.id, obj.workspace_id)
if str(user.id) in obj.allowed_users:
actions |= {'VIEW', 'USE'}
data['actions'] = sorted(actions)
data['can_delete'] = 'EDIT' in actions
if isinstance(obj, AIParserResource):
data.update(enabled=obj.enabled, allowed_users=obj.allowed_users,
options=obj.options, credential_configured=bool(obj.credential))
# Only callers of admin management endpoints receive endpoint addresses.
elif isinstance(obj, AIFlow):
data.update(revision=obj.revision, nodes=obj.nodes)
elif isinstance(obj, AIService):
from system_manage.ai_hub_service_api import get_config
data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users,
requests_per_minute=obj.requests_per_minute,
key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run', api_config=get_config(obj.id))
elif isinstance(obj, AITask):
from system_manage.ai_hub_progress import task_progress
data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt,
started_at=obj.started_at, finished_at=obj.finished_at)
if 'snapshot' not in obj.get_deferred_fields():
data['artifact_id'] = (obj.snapshot.get('workbench_artifact') or {}).get('id')
if detail:
data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj))
if any(n.get('type') == 'organize' for n in obj.snapshot.get('nodes', [])):
data['summary_input'] = {key: obj.input_data[key] for key in ('markdown', 'heading_structure') if key in obj.input_data}
elif isinstance(obj, AIArtifact):
data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None,
task_status=obj.task.status if obj.task_id else None, task_error=obj.task.error if obj.task_id else '')
meta = next((x for x in reversed(obj.history or []) if isinstance(x, dict) and x.get('_type') == 'workbench_tasks_v1'), {})
task_ids = [meta.get('parse_task_id'), meta.get('summary_task_id')]
states = {str(pk): status for pk, status in AITask.objects.filter(
id__in=[x for x in task_ids if x]).values_list('id', 'status')}
conflict = bool(obj.task_id and meta.get('conflict_task_id') == str(obj.task_id))
if conflict:
states[str(obj.task_id)] = 'CONFLICT'
data.update(file_name=meta.get('file_name', ''), parse_status=states.get(task_ids[0]),
summary_status=states.get(task_ids[1]), task_result_conflict=conflict)
if detail:
from system_manage.heading_structure import artifact_state, version_history, summary_state
data.update(original=obj.original, markdown=obj.markdown, history=version_history(obj), heading_structure=artifact_state(obj), summary_state=summary_state(obj))
return data
def parser_save(workspace, user, data, obj=None):
if obj is None:
require_admin(user)
else:
if set(data) - {'allowed_users'}:
require_resource_action(user, obj, 'EDIT')
if 'allowed_users' in data:
require_resource_action(user, obj, 'AUTHORIZE')
endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/')
parsed = urlsplit(endpoint)
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValidationError('解析服务地址无效')
obj = obj or AIParserResource(workspace_id=workspace, owner_id=user.id)
obj.name = text(data.get('name', obj.name), '名称', limit=128)
obj.endpoint = endpoint
if 'api_key' in data:
from common.utils.rsa_util import rsa_long_encrypt
key = text(data['api_key'], '密钥', required=False, limit=4096)
obj.credential = rsa_long_encrypt(key) if key else ''
enabled = data.get('enabled', obj.enabled)
if not isinstance(enabled, bool):
raise ValidationError('启用状态无效')
obj.enabled = enabled
if 'allowed_users' in data:
obj.allowed_users = users(data['allowed_users'])
options = data.get('options', obj.options)
from system_manage.parsers.catalog import validate_options
obj.options = validate_options(options, endpoint)
obj.save()
return obj
def parser_test(parser):
"""Probe the registered service API, without submitting files or returning service content."""
import time
import requests
from urllib.parse import urlunsplit
from common.utils.rsa_util import rsa_long_decrypt
parsed = urlsplit(parser.endpoint.rstrip('/'))
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password:
raise ValidationError('解析服务地址无效')
base_path = parsed.path
if parser.options.get('protocol') == 'v1':
if base_path.endswith('/v1'):
base_path = base_path[:-3]
url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', ''))
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
started = time.monotonic()
try:
response = requests.get(url, headers=headers, timeout=(3, 7), allow_redirects=False)
if response.status_code != 200:
raise ValidationError(f'连接测试失败(HTTP {response.status_code})')
data = response.json()
except requests.RequestException as exc:
raise ValidationError('解析服务连接失败或超时') from exc
except ValueError as exc:
raise ValidationError('解析服务未返回有效接口定义') from exc
paths = data.get('paths') if isinstance(data, dict) else None
v1 = parser.options.get('protocol') == 'v1'
required = ('/v1/uploads', '/v1/parse/jobs') if v1 else ('/file_parse',)
if not isinstance(paths, dict) or not all(any(path.endswith(suffix) and 'post' in methods
for path, methods in paths.items() if isinstance(methods, dict)) for suffix in required):
raise ValidationError('服务接口与所选 MinerU 版本不匹配')
# Existence of the API does not prove that a local GPU/model backend is ready.
return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000),
'api_version': data.get('info', {}).get('version', ''), 'message': '解析接口连接正常'}
def validate_nodes(workspace, user, nodes):
if not isinstance(nodes, list) or not 1 <= len(nodes) <= 3:
raise ValidationError('请选择处理节点')
types = [n.get('type') for n in nodes if isinstance(n, dict)]
if len(types) != len(nodes) or types[-1] != 'render' or len(set(types)) != len(types):
raise ValidationError('流程须以导图生成为终点,节点不可重复')
if types != [t for t in ('parse', 'organize', 'render') if t in types]:
raise ValidationError('节点顺序应为解析、整理、导图生成')
result = []
for node in nodes:
kind = node['type']
clean = {'type': kind}
if kind == 'parse':
if node.get('processor') == 'builtin':
result.append({'type': 'parse', 'processor': 'builtin'})
continue
parser = one('parsers', node.get('resource_id'), workspace, user, lock=connection.in_atomic_block)
require_use(user, parser)
if not parser.enabled:
raise ValidationError('解析服务已停用')
clean['resource_id'] = str(parser.id)
if kind == 'organize':
if node.get('operation') not in (None, 'heading_repair'):
raise ValidationError('整理操作无效')
if node.get('operation'):
clean['operation'] = node['operation']
clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id)
clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000)
from system_manage.summary_plan import validate_options
from system_manage.ai_hub_organize import resolve_settings
from models_provider.tools import get_model_default_params
from system_manage.mcp_business import summary_model_defaults, summary_supported_parameters
try:
clean.update(validate_options(node))
model = model_for(workspace, user, clean['model_id'])
resolve_settings(summary_model_defaults(model, get_model_default_params(model)), clean)
if clean.get('top_p') is not None and 'top_p' not in summary_supported_parameters(model):
raise ValueError('该模型未声明支持 top_p 参数')
except ValueError as exc:
raise ValidationError(str(exc)) from None
result.append(clean)
return result
@transaction.atomic
def flow_create(workspace, user, data):
nodes = validate_nodes(workspace, user, data.get('nodes'))
# Every save is immutable; published services always reference a fixed row.
previous = one('flows', data['previous_id'], workspace, user, edit=True, lock=True) if data.get('previous_id') else None
return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name'), '名称', limit=128), nodes=nodes,
revision=previous.revision + 1 if previous else 1)
@transaction.atomic
def service_create(workspace, user, data):
require_admin(user)
flow = one('flows', data.get('flow_id'), workspace, user, lock=True)
validate_nodes(workspace, user, flow.nodes)
obj = AIService.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name'), '名称', limit=128), flow=flow,
allowed_users=users(data.get('allowed_users', [])),
requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率'))
if 'api_config' in data:
from system_manage.ai_hub_service_api import save_config
save_config(obj.id, data['api_config'])
return obj
def make_snapshot(workspace, user, nodes):
clean = validate_nodes(workspace, user, nodes)
for node in clean:
if node['type'] == 'parse':
if node.get('processor') == 'builtin':
continue
parser = one('parsers', node['resource_id'], workspace, user)
node['options'] = dict(parser.options)
node['resource_version'] = parser.updated_at.isoformat()
if node['type'] == 'organize':
model = model_for(workspace, user, node['model_id'])
node.update(model_name=model.model_name, provider=model.provider, resource_version=model.update_time.isoformat())
return clean
@transaction.atomic
def submit(workspace, user, data, upload=None, service=None, request=None):
from users.models import User
current_user = User.objects.select_for_update().get(id=user.id)
if not current_user.is_active:
raise PermissionDenied('用户已停用')
if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
raise ValidationError('待处理任务已达上限')
if service:
service = one('services', service.pk, workspace, user, lock=True)
require_use(user, service)
if not service.enabled:
raise ValidationError('服务已停用')
nodes = service.flow.nodes
else:
nodes = one('flows', data['flow_id'], workspace, user, lock=True).nodes if data.get('flow_id') else data.get('nodes')
snapshot = make_snapshot(workspace, user, nodes)
from system_manage.mcp_registry import pipeline_resources, resolve
required = pipeline_resources(snapshot)
for key in required: resolve(workspace, user, key)
priority = integer(data.get('priority', 5), 0, 9, '优先级')
from system_manage.access_control import enforce_task
enforce_task(request, service.id if service else None, snapshot)
markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT)
if upload and upload.size > MAX_FILE:
raise ValidationError('文件不能超过20MB')
if upload and snapshot[0]['type'] != 'parse':
raise ValidationError('文件任务需要解析节点')
if not upload and snapshot[0]['type'] == 'parse':
raise ValidationError('请选择文件')
if not upload and not markdown.strip():
raise ValidationError('文档内容不能为空')
from system_manage.ai_hub_assets import asset_lock, normalize
with asset_lock():
markdown = text(normalize(markdown, workspace, user), '文档内容', required=False)
heading_input = {}
if data.get('heading_structure') is not None:
from system_manage.heading_structure import analyze, validate_manual
if not isinstance(data['heading_structure'], dict):
raise ValidationError('层级数据无效')
try:
_, structure = analyze(markdown, previous=data['heading_structure'], repair=False)
structure = validate_manual(structure, data['heading_structure'])
except ValueError as exc:
raise ValidationError(str(exc)) from None
heading_input = {'heading_structure': structure, 'original': text(data.get('original', markdown), '原稿', required=False)}
if data.get('_heading_source') is not None:
heading_input['heading_source'] = data['_heading_source']
# Plan without invoking a model. Reserve only useful internal parallel calls.
from system_manage.mcp_registry import settings_row
from system_manage.mcp_business import summary_model_defaults, summary_counter
from system_manage.summary_plan import token_counter
from system_manage.ai_hub_organize import plan_document, resolve_settings
from models_provider.tools import get_model_default_params
from system_manage.ai_hub_assets import protect_images
scheduler_settings = settings_row().meta
capacity = scheduler_settings.get('capacity', 4)
worker_capacity = max(1, capacity - min(scheduler_settings.get('interactive_reserve', 1), max(0, capacity - 1)))
resource_slots = {}
for node in snapshot:
if node['type'] != 'organize' or node.get('operation') == 'heading_repair':
continue
key = 'model:' + node['model_id']
slots = min(node.get('concurrency', 1) if node.get('parallel', True) else 1,
resolve(workspace, user, key)[0]['capacity'], worker_capacity)
if snapshot[0]['type'] != 'parse':
model = model_for(workspace, user, node['model_id'])
defaults = summary_model_defaults(model, get_model_default_params(model))
try:
options = {**node, 'max_input_tokens': min(int(defaults.get('max_input_tokens', 2000000)), node.get('max_input_tokens', 2000000))}
if 'input_tokens' not in options and defaults.get('input_tokens'):
options['input_tokens'] = int(defaults['input_tokens'])
settings = resolve_settings(defaults, node)
count, _ = summary_counter(model, defaults, markdown, workspace, user.id, settings, options)
if node.get('structure_mode', 'preserve') in ('preserve', 'reorganize'):
from system_manage.summary_sections import build_plan
planned = build_plan(protect_images(markdown)[0], node['prompt'], settings, options, count)
slots = min(slots, max(1, len(planned['batches'])))
else:
_, _, _, _, chunks = plan_document(protect_images(markdown)[0], node['prompt'], settings, options, count)
slots = min(slots, len(chunks))
except ValueError as exc:
raise ValidationError(str(exc)) from None
resource_slots[key] = slots
task = AITask.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service,
snapshot={'nodes': snapshot, 'resources': required, 'resource_slots': resource_slots, 'priority': priority, 'flow_id': str(service.flow_id) if service else data.get('flow_id'),
'flow_revision': service.flow.revision if service else None},
input_data={'markdown': markdown, **heading_input}, file_name=upload.name[:255] if upload else '',
file_data=upload.read() if upload else None)
if 'workbench_artifact' in data:
spec = data['workbench_artifact']
if not isinstance(spec, dict) or set(spec) - {'id', 'revision', 'layout'}:
raise ValidationError('导图任务参数无效')
if spec.get('id'):
from rest_framework.serializers import UUIDField
artifact_id = UUIDField().run_validation(spec['id'])
artifact = AIArtifact.objects.select_for_update().filter(
pk=artifact_id, workspace_id=workspace, owner_id=user.id).first()
if not artifact:
raise ValidationError('导图不存在或无权访问')
if type(spec.get('revision')) is not int or spec['revision'] != artifact.revision:
raise ValidationError('内容已更新,请重新打开')
else:
from users.models import User
from system_manage.artifact_settings import limit
User.objects.select_for_update().get(pk=user.id)
maximum = limit()
if AIArtifact.objects.filter(workspace_id=workspace, owner_id=user.id).count() >= maximum:
raise ValidationError(f'最多可保存 {maximum} 张导图,请先删除不需要的导图')
layout = spec.get('layout', 'tree')
if layout not in ('tree', 'radial', 'horizontal', 'vertical'):
raise ValidationError('布局无效')
artifact = AIArtifact.objects.create(workspace_id=workspace, owner_id=user.id,
name=task.name, layout=layout, original=heading_input.get('original', markdown), markdown=markdown)
artifact.task = task
meta = next((x.copy() for x in reversed(artifact.history or []) if isinstance(x, dict) and x.get('_type') == 'workbench_tasks_v1'), {'_type': 'workbench_tasks_v1'})
meta.pop('conflict_task_id', None)
is_summary = any(node['type'] == 'organize' for node in snapshot)
meta['summary_task_id' if is_summary else 'parse_task_id'] = str(task.id)
if upload:
meta['file_name'] = upload.name[:255]
artifact.history = [x for x in (artifact.history or []) if not isinstance(x, dict) or x.get('_type') != 'workbench_tasks_v1'] + [meta]
artifact.save(update_fields=['task', 'history', 'updated_at'])
task.snapshot['workbench_artifact'] = {'id': str(artifact.id), 'revision': artifact.revision}
task.save(update_fields=['snapshot', 'updated_at'])
return task
@transaction.atomic
def artifact_rename(obj, data):
if set(data) != {'action', 'name', 'revision'}:
raise ValidationError('重命名参数无效')
if isinstance(data['revision'], bool) or not isinstance(data['revision'], int) or data['revision'] != obj.revision:
raise ValidationError('内容已更新,请重新打开')
obj.name = text(data['name'], '名称', limit=128)
obj.save(update_fields=['name', 'updated_at'])
return obj
@transaction.atomic
def artifact_save(workspace, user, data, obj=None):
from system_manage.heading_structure import analyze, artifact_state, store_artifact_state, store_summary_state, summary_state, validate_manual
previous_structure = artifact_state(obj) if obj else {}
if obj:
obj = AIArtifact.objects.select_for_update().get(pk=obj.pk)
previous_structure = artifact_state(obj)
if data.get('revision') != obj.revision:
raise ValidationError('内容已更新,请重新打开')
obj.revision += 1
else:
from users.models import User
from system_manage.artifact_settings import limit
User.objects.select_for_update().get(pk=user.id)
maximum = limit()
if AIArtifact.objects.filter(workspace_id=workspace, owner_id=user.id).count() >= maximum:
raise ValidationError(f'最多可保存 {maximum} 张导图,请先删除不需要的导图')
task = one('tasks', data['task_id'], workspace, user, lock=True) if data.get('task_id') else None
if task and task.status != 'SUCCESS':
raise ValidationError('任务尚未成功')
obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task,
original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT))
if task:
previous_structure = (task.result.get('source') or {}).get('heading_structure', {})
obj.name = text(data.get('name'), '名称', limit=128)
obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT)
obj.layout = data.get('layout', 'tree')
if obj.layout not in ('tree', 'radial', 'horizontal', 'vertical'):
raise ValidationError('布局无效')
from system_manage.ai_hub_assets import asset_lock, normalize
with asset_lock():
obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容')
obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False)
if 'summary_state' in data:
state = data['summary_state']
if not isinstance(state, dict) or set(state) - {'source_markdown', 'smart_markdown', 'mode'}:
raise ValidationError('智能总结内容无效')
if state.get('mode', 'source') not in ('source', 'smart'):
raise ValidationError('智能总结状态无效')
clean_state = {'mode': state.get('mode', 'source')}
for key in ('source_markdown', 'smart_markdown'):
clean_state[key] = text(normalize(text(state.get(key, ''), '总结内容', required=False, limit=MAX_INLINE_TEXT), workspace, user), '总结内容', required=False, limit=MAX_INLINE_TEXT)
store_summary_state(obj, clean_state)
elif summary_state(obj):
state = dict(summary_state(obj))
state.pop('_type', None)
state['smart_markdown' if state.get('mode') == 'smart' else 'source_markdown'] = obj.markdown
store_summary_state(obj, state)
for entry in obj.history:
if 'markdown' in entry:
entry['markdown'] = normalize(entry['markdown'], workspace, user)
incoming = data.get('heading_structure')
if incoming is not None or previous_structure:
try:
_, structure = analyze(obj.markdown, previous=previous_structure or incoming, repair=False)
if incoming is not None:
structure = validate_manual(structure, incoming)
_, structure = analyze(obj.markdown, previous=structure, repair=False)
except ValueError as exc:
raise ValidationError(str(exc)) from None
store_artifact_state(obj, structure)
obj.save()
return obj
@transaction.atomic
def delete_record(kind, pk, workspace, user):
obj = one(kind, pk, workspace, user, edit=True, lock=True)
if kind == 'parsers':
reference = [{'type': 'parse', 'resource_id': str(obj.id)}]
if AIFlow.objects.filter(workspace_id=workspace, nodes__contains=reference).exists():
raise ValidationError('解析服务已被流程引用,请先删除相关流程')
if AITask.objects.filter(workspace_id=workspace, status__in=['PENDING', 'RUNNING'],
snapshot__nodes__contains=reference).exists():
raise ValidationError('解析服务有未完成任务,请稍后删除')
elif kind == 'flows':
if AIService.objects.filter(flow=obj).exists():
raise ValidationError('流程已被服务引用,请先删除相关服务')
elif kind == 'services':
if AITask.objects.filter(service=obj, status__in=['PENDING', 'RUNNING']).exists():
raise ValidationError('服务有未完成任务,请稍后删除')
elif kind == 'tasks':
if obj.status in ('PENDING', 'RUNNING'):
raise ValidationError('待处理或运行中的任务不能删除')
elif kind != 'artifacts':
raise ValidationError('资源类型无效')
# Historical task snapshots and saved artifacts remain independent records.
resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind)
if resource_type:
from system_manage.models.platform_access import PlatformGrant
PlatformGrant.objects.filter(workspace_id=workspace, resource_type=resource_type,
target=str(obj.id)).delete()
if kind == 'services':
from system_manage.ai_hub_service_api import save_config
save_config(obj.id)
obj.delete()
def parse_file(parser, name, content, options=None, progress_callback=None):
import requests
from common.utils.rsa_util import rsa_long_decrypt
opts = options if options is not None else parser.options
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
from system_manage.parsers import parse
return parse(parser, name, content, opts, headers, progress_callback=progress_callback)
def markdown_tree(markdown, excluded_lines=None):
"""Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。"""
import re
excluded_lines = set(excluded_lines or [])
root = {'id': 'root', 'text': '文档', 'children': []}
heading_stack = [(0, root)]
list_stack = []
fence = None
table_buffer, table_start = [], None
def container():
return list_stack[-1][1] if list_stack else heading_stack[-1][1]
def flush_table():
nonlocal table_buffer, table_start
if not table_buffer:
return
index, lines = table_start, table_buffer
table_buffer, table_start = [], None
def cells(raw):
return [cell.strip() for cell in raw.strip().strip('|').split('|')]
if len(lines) >= 2 and re.match(r'^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$', lines[1]):
headers = cells(lines[0])
aligns = []
for cell in cells(lines[1]):
left, right = cell.startswith(':'), cell.endswith(':')
aligns.append('center' if left and right else 'right' if right else 'left')
rows = [cells(raw) for raw in lines[2:]]
node = {'id': f'n{index}', 'type': 'table', 'text': '表格:' + '、'.join(headers),
'table': {'headers': headers, 'aligns': aligns, 'rows': rows,
'row_count': len(rows), 'column_count': len(headers)},
'children': []}
container()['children'].append(node)
else:
for offset, raw in enumerate(lines):
container()['children'].append({'id': f'n{index + offset}', 'text': raw.strip(), 'children': []})
list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$')
for index, line in enumerate(markdown.splitlines()):
if excluded_lines and index in excluded_lines:
continue
fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
if fence_match:
token = fence_match.group(1)
if fence is None:
fence = token
flush_table()
list_stack.clear()
elif token[0] == fence[0] and len(token) >= len(fence):
fence = None
continue
if fence:
continue
stripped = line.strip()
if stripped.startswith('|') and stripped.count('|') >= 2:
if table_start is None:
table_start = index
list_stack.clear()
table_buffer.append(line)
continue
flush_table()
if not stripped:
continue
heading = re.match(r'^\s{0,3}(#{1,6})\s+(.+)', line)
if heading:
level = len(heading.group(1))
list_stack.clear()
while heading_stack[-1][0] >= level:
heading_stack.pop()
node = {'id': f'n{index}', 'text': heading.group(2).rstrip('# ').strip(), 'children': []}
heading_stack[-1][1]['children'].append(node)
heading_stack.append((level, node))
continue
item = list_pattern.match(line)
if item:
indent = len(item.group(1).expandtabs(4))
while list_stack and list_stack[-1][0] >= indent:
list_stack.pop()
ordered = item.group(3)
node = {'id': f'n{index}', 'type': 'list-item',
'text': item.group(4).strip(),
'list': {'type': 'ol' if ordered else 'ul', 'index': int(ordered) if ordered else None},
'children': []}
container()['children'].append(node)
list_stack.append((indent, node))
continue
if line[:1].isspace() and list_stack:
list_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
continue
list_stack.clear()
heading_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
flush_table()
return root