diff --git a/apps/application/sql/list_application_user.sql b/apps/application/sql/list_application_user.sql index a1a37e3fa..9923ca0d7 100644 --- a/apps/application/sql/list_application_user.sql +++ b/apps/application/sql/list_application_user.sql @@ -16,6 +16,6 @@ from (select application."id"::text, application."name", from application left join "user" on user_id = "user".id where application."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list))) temp ${application_query_set} diff --git a/apps/application/sql/list_application_user_ee.sql b/apps/application/sql/list_application_user_ee.sql index bfab07b33..6601376a6 100644 --- a/apps/application/sql/list_application_user_ee.sql +++ b/apps/application/sql/list_application_user_ee.sql @@ -16,7 +16,7 @@ from (select application."id"::text, application."name", from application left join "user" on user_id = "user".id where "application".id::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then 'ROLE' = any (permission_list) @@ -28,9 +28,9 @@ from (select application."id"::text, application."name", from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id = role_permission.role_id - where user_role_relation.user_id = workspace_user_resource_permission.user_id + where user_role_relation.user_id = effective_resource_permission.user_id and user_role_relation.workspace_id = - workspace_user_resource_permission.workspace_id) + effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) diff --git a/apps/application/views/application.py b/apps/application/views/application.py index 518b74c34..cee7357e8 100644 --- a/apps/application/views/application.py +++ b/apps/application/views/application.py @@ -272,6 +272,8 @@ class McpServers(APIView): CompareConstants.AND), RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) def post(self, request: Request, workspace_id, application_id: str): + from system_manage.access_control import enforce + enforce(request, 'application', application_id) return result.success(ApplicationOperateSerializer( data={'mcp_servers': request.query_params.get('mcp_servers'), 'workspace_id': workspace_id, 'user_id': request.user.id, diff --git a/apps/application/views/application_chat.py b/apps/application/views/application_chat.py index d60206ded..66c85d827 100644 --- a/apps/application/views/application_chat.py +++ b/apps/application/views/application_chat.py @@ -132,6 +132,8 @@ class OpenView(APIView): CompareConstants.AND), RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) def get(self, request: Request, workspace_id: str, application_id: str): + from system_manage.access_control import enforce + enforce(request, 'application', application_id) return result.success(OpenChatSerializers( data={'workspace_id': workspace_id, 'application_id': application_id, 'chat_user_id': str(uuid.uuid7()), 'chat_user_type': ChatUserType.ANONYMOUS_USER, @@ -152,7 +154,7 @@ class ChatView(APIView): tags=[_('Application')] # type: ignore ) def post(self, request: Request, chat_id: str): - return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data) + return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data, request=request) class PromptGenerateView(APIView): authentication_classes = [TokenAuth] @@ -176,4 +178,4 @@ class PromptGenerateView(APIView): @log(menu='Application', operate='Generate prompt', get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id'))) def post(self, request: Request, workspace_id: str, model_id:str, application_id: str): - return PromptGenerateSerializer(data={'workspace_id': workspace_id, 'model_id': model_id, 'application_id': application_id}).generate_prompt(instance=request.data) \ No newline at end of file + return PromptGenerateSerializer(data={'workspace_id': workspace_id, 'model_id': model_id, 'application_id': application_id}).generate_prompt(instance=request.data) diff --git a/apps/chat/serializers/chat.py b/apps/chat/serializers/chat.py index 28e361292..c05a517dc 100644 --- a/apps/chat/serializers/chat.py +++ b/apps/chat/serializers/chat.py @@ -131,10 +131,12 @@ def get_post_handler(chat_info: ChatInfo): class DebugChatSerializers(serializers.Serializer): chat_id = serializers.UUIDField(required=True, label=_("Conversation ID")) - def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse()): + def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse(), request=None): self.is_valid(raise_exception=True) chat_id = self.data.get('chat_id') chat_info: ChatInfo = ChatInfo.get_cache(chat_id) + from system_manage.access_control import enforce + enforce(request, 'application', chat_info.application_id) application = QuerySet(Application).filter(id=chat_info.application_id).first() chat_info.application = application return ChatSerializers(data={ diff --git a/apps/chat/serializers/chat_authentication.py b/apps/chat/serializers/chat_authentication.py index 7d9c18e74..decba655b 100644 --- a/apps/chat/serializers/chat_authentication.py +++ b/apps/chat/serializers/chat_authentication.py @@ -86,6 +86,8 @@ class AnonymousAuthenticationSerializer(serializers.Serializer): access_token = self.data.get("access_token") application_access_token = QuerySet(ApplicationAccessToken).filter(access_token=access_token).first() if application_access_token is not None and application_access_token.is_active: + from system_manage.access_control import enforce + enforce(request, 'application', application_access_token.application_id) api_key = self.data.get('api_key') or self.get_request_api_key(request) if api_key: application_api_key = self.validate_application_api_key(application_access_token.application_id, api_key) @@ -121,6 +123,8 @@ class AuthProfileSerializer(serializers.Serializer): if not application_access_token.is_active: raise NotFound404(404, _("Invalid access_token")) application_id = application_access_token.application_id + from system_manage.access_control import enforce + enforce(request, 'application', application_id) api_key = self.data.get("api_key") or AnonymousAuthenticationSerializer.get_request_api_key(request) if api_key: AnonymousAuthenticationSerializer.validate_application_api_key(application_id, api_key) diff --git a/apps/common/auth/authenticate.py b/apps/common/auth/authenticate.py index a762fed11..a282f7aa6 100644 --- a/apps/common/auth/authenticate.py +++ b/apps/common/auth/authenticate.py @@ -85,7 +85,17 @@ class TokenAuth(TokenAuthentication): token_details = TokenDetails(token) for handle in handles: if handle.support(request, token, token_details.get_token_details): - return handle.handle(request, token, token_details.get_token_details) + authenticated = handle.handle(request, token, token_details.get_token_details) + from common.constants.permission_constants import ChatAuth + if isinstance(authenticated[1], ChatAuth): + from system_manage.access_control import enforce + match = getattr(request, 'resolver_match', None) + routed_id = match.kwargs.get('application_id') if match else None + if routed_id and str(routed_id) != str(authenticated[1].application_id): + from common.exception.app_exception import AppUnauthorizedFailed + raise AppUnauthorizedFailed(403, '无权访问该助手') + enforce(request, 'application', authenticated[1].application_id) + return authenticated raise AppAuthenticationFailed(1002, _('Authentication information is incorrect! illegal user')) except Exception as e: traceback.print_stack() diff --git a/apps/common/auth/handle/impl/user_token.py b/apps/common/auth/handle/impl/user_token.py index f34136547..4e066dca1 100644 --- a/apps/common/auth/handle/impl/user_token.py +++ b/apps/common/auth/handle/impl/user_token.py @@ -275,6 +275,10 @@ def get_role_list(user, def get_auth(user): + from system_manage.platform_access import is_admin + # The old field is a compatibility projection, never an authority. + administrator = is_admin(user) + user.role = 'ADMIN' if administrator else 'USER' workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping") workspace_model = DatabaseModelManage.get_model("workspace_model") role_model = DatabaseModelManage.get_model("role_model") @@ -284,7 +288,15 @@ def get_auth(user): role_model, role_permission_mapping_model) role_list = get_role_list(user, workspace_user_role_mapping_model, workspace_model, role_model, role_permission_mapping_model) - return Auth(role_list, permission_list) + if workspace_user_role_mapping_model is None: + role_list = ['ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')] if administrator else ['USER', get_role_permission(RoleConstants.USER, 'default')] + elif administrator: + role_list = list(set(role_list) | {'ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')}) + else: + role_list = [role for role in role_list if role not in ('ADMIN', 'EXTENDS_ADMIN')] + from system_manage.platform_access import permission_strings + # Evaluate new grants live, independently of legacy cached permissions. + return Auth(role_list, list(set(permission_list) | permission_strings(user))) class UserToken(AuthBaseHandle): @@ -306,4 +318,6 @@ class UserToken(AuthBaseHandle): if not user.is_active or user.password != cache_token.password: raise AppAuthenticationFailed(1002, _('Authentication information is incorrect')) auth = get_auth(user) + from system_manage.platform_access import check_feature_request + check_feature_request(request, user) return user, auth diff --git a/apps/folders/serializers/folder.py b/apps/folders/serializers/folder.py index 790a913ca..bca0772a2 100644 --- a/apps/folders/serializers/folder.py +++ b/apps/folders/serializers/folder.py @@ -16,7 +16,7 @@ from folders.api.folder import FolderCreateRequest from knowledge.models import KnowledgeFolder, Knowledge from knowledge.serializers.knowledge import KnowledgeSerializer from knowledge.serializers.knowledge_folder import KnowledgeFolderTreeSerializer -from system_manage.models import WorkspaceUserResourcePermission +from system_manage.models import WorkspaceUserResourcePermission, EffectiveResourcePermission from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer from tools.models import ToolFolder, Tool from tools.serializers.tool import ToolSerializer @@ -101,7 +101,7 @@ def get_max_depth(current_node): def has_target_permission(workspace_id, source, user_id, target): - return QuerySet(WorkspaceUserResourcePermission).filter(workspace_id=workspace_id, user_id=user_id, + return QuerySet(EffectiveResourcePermission).filter(workspace_id=workspace_id, user_id=user_id, auth_target_type=source, target=target, permission_list__contains=['MANAGE']).exists() @@ -249,13 +249,13 @@ class FolderSerializer(serializers.Serializer): .annotate(id_str=Cast('id', TextField())) .values_list('id_str', flat=True)) # 检查文件夹是否存在未授权当前用户的资源 - auth_list = QuerySet(WorkspaceUserResourcePermission).filter( + auth_list = QuerySet(EffectiveResourcePermission).filter( Q(workspace_id=self.data.get('workspace_id')) & Q(user_id=self.data.get('user_id')) & Q(auth_target_type=self.data.get('source')) & Q(target__in=source_ids) & Q(permission_list__overlap=[ResourcePermission.MANAGE, ResourcePermissionRole.ROLE]) - ).count() + ).values('target').distinct().count() if auth_list != len(source_ids): raise AppApiException(500, _('This folder contains resources that you dont have permission')) self.delete_source(node) @@ -319,7 +319,7 @@ class FolderTreeSerializer(serializers.Serializer): if name is not None: base_q &= Q(name__contains=name) if not workspace_manage: - base_q &= (Q(id__in=WorkspaceUserResourcePermission.objects.filter(user_id=current_user.id, + base_q &= (Q(id__in=EffectiveResourcePermission.objects.filter(user_id=current_user.id, auth_target_type=self.data.get('source'), workspace_id=self.data.get( 'workspace_id'), diff --git a/apps/knowledge/serializers/document.py b/apps/knowledge/serializers/document.py index 0df156081..528920349 100644 --- a/apps/knowledge/serializers/document.py +++ b/apps/knowledge/serializers/document.py @@ -1066,6 +1066,13 @@ class DocumentSerializers(serializers.Serializer): raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度") file_list = instance.get("file") + from system_manage.access_control import enforce + from system_manage.parsers import extension, MINERU_INPUTS + request = self.context.get('request') + enforce(request, 'knowledge', self.data.get('knowledge_id')) + parser_id = (knowledge.meta or {}).get('parser_id') if knowledge else None + if parser_id and any(extension(file.name) in MINERU_INPUTS for file in file_list): + enforce(request, 'parser', parser_id) return reduce( lambda x, y: [*x, *y], [self.file_to_paragraph( diff --git a/apps/knowledge/sql/list_knowledge_user.sql b/apps/knowledge/sql/list_knowledge_user.sql index 55c638a97..82b019c76 100644 --- a/apps/knowledge/sql/list_knowledge_user.sql +++ b/apps/knowledge/sql/list_knowledge_user.sql @@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, FROM (SELECT knowledge.* FROM knowledge knowledge ${knowledge_custom_sql} AND id::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list))) temp_knowledge LEFT JOIN (SELECT "count"("id") AS document_count, "sum"("char_length") "char_length", knowledge_id FROM "document" diff --git a/apps/knowledge/sql/list_knowledge_user_ee.sql b/apps/knowledge/sql/list_knowledge_user_ee.sql index 7f76d8a5e..2e22d7b24 100644 --- a/apps/knowledge/sql/list_knowledge_user_ee.sql +++ b/apps/knowledge/sql/list_knowledge_user_ee.sql @@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, FROM (SELECT knowledge.* FROM knowledge knowledge ${knowledge_custom_sql} AND "knowledge".id::text in (select target - from workspace_user_resource_permission + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then @@ -34,8 +34,8 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id=role_permission.role_id - where user_role_relation.user_id=workspace_user_resource_permission.user_id - and user_role_relation.workspace_id=workspace_user_resource_permission.workspace_id) + where user_role_relation.user_id=effective_resource_permission.user_id + and user_role_relation.workspace_id=effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) end diff --git a/apps/knowledge/views/document.py b/apps/knowledge/views/document.py index b37800899..f4edf1255 100644 --- a/apps/knowledge/views/document.py +++ b/apps/knowledge/views/document.py @@ -194,7 +194,7 @@ class DocumentView(APIView): return result.success(DocumentSerializers.Split(data={ 'workspace_id': workspace_id, 'knowledge_id': knowledge_id, - }, context={'user': request.user}).parse(split_data)) + }, context={'user': request.user, 'request': request}).parse(split_data)) class SplitPattern(APIView): authentication_classes = [TokenAuth] diff --git a/apps/models_provider/sql/list_model_user.sql b/apps/models_provider/sql/list_model_user.sql index 15dbf21d7..f9a8faa68 100644 --- a/apps/models_provider/sql/list_model_user.sql +++ b/apps/models_provider/sql/list_model_user.sql @@ -15,5 +15,5 @@ FROM (SELECT model."id"::text, model."name", from model left join "user" on user_id = "user".id where model."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list)) ) temp ${model_query_set} diff --git a/apps/models_provider/sql/list_model_user_ee.sql b/apps/models_provider/sql/list_model_user_ee.sql index 5febcc418..5b82da1d9 100644 --- a/apps/models_provider/sql/list_model_user_ee.sql +++ b/apps/models_provider/sql/list_model_user_ee.sql @@ -15,7 +15,7 @@ FROM (SELECT model."id"::text, model."name", from model left join "user" on user_id = "user".id where model."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then 'ROLE' = any (permission_list) @@ -27,9 +27,9 @@ FROM (SELECT model."id"::text, model."name", from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id = role_permission.role_id - where user_role_relation.user_id = workspace_user_resource_permission.user_id + where user_role_relation.user_id = effective_resource_permission.user_id and user_role_relation.workspace_id = - workspace_user_resource_permission.workspace_id) + effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) diff --git a/apps/system_manage/access_control.py b/apps/system_manage/access_control.py new file mode 100644 index 000000000..9effeb1be --- /dev/null +++ b/apps/system_manage/access_control.py @@ -0,0 +1,235 @@ +"""Request-side access policy enforcement, using existing settings and audit tables.""" +import ipaddress +import os +from datetime import timedelta +from zoneinfo import ZoneInfo + +from django.db import transaction +from django.utils import timezone +from rest_framework.exceptions import ValidationError +from common.exception.app_exception import AppUnauthorizedFailed +from system_manage.models import AccessControlPolicy, AccessControlPolicyApplicationRecord, SystemSetting +from system_manage.models.log_management import Log + +# A separate settings row; no schema change or automatic migration is needed. +SETTING_ID = 20 +POLICY_TIMEZONE = ZoneInfo('Asia/Shanghai') +DEVICE_TYPES = ('桌面端', '移动终端', '平板', '服务器') + + +def configuration(lock=False): + query = SystemSetting.objects + if lock: + query.get_or_create(type=SETTING_ID, defaults={'meta': {'bindings': [], 'region_networks': []}}) + query = query.select_for_update() + row = query.filter(type=SETTING_ID).first() + return row, dict(row.meta or {}) if row else {'bindings': [], 'region_networks': []} + + +def network(value): + try: + return str(ipaddress.ip_network(value.strip(), strict=False)) + except (ValueError, AttributeError): + raise ValidationError('请输入有效的 IP 或网段') + + +def client_ip(request): + """Only consume forwarded addresses from explicitly trusted proxy chains.""" + try: + peer = ipaddress.ip_address(request.META.get('REMOTE_ADDR', '')) + except ValueError: + return '' + trusted = [] + for value in os.environ.get('MAXKB_ACCESS_TRUSTED_PROXIES', '').split(','): + if value.strip(): + trusted.append(ipaddress.ip_network(value.strip(), strict=False)) + forwarded = request.META.get('HTTP_X_FORWARDED_FOR', '') + if any(peer in block for block in trusted) and forwarded: + for value in reversed(forwarded.split(',')): + if not any(peer in block for block in trusted): + break + try: + peer = ipaddress.ip_address(value.strip()) + except ValueError: + raise AppUnauthorizedFailed(403, '接入受限:来源 IP 无效') + return str(peer) + + +def device_type(request): + agent = request.META.get('HTTP_USER_AGENT', '').lower() + if any(token in agent for token in ('ipad', 'tablet')) or ('android' in agent and 'mobile' not in agent): + return '平板' + if any(token in agent for token in ('mobile', 'iphone', 'android')): + return '移动终端' + if any(token in agent for token in ('mozilla', 'windows', 'macintosh', 'x11')): + return '桌面端' + if any(token in agent for token in ('curl', 'python', 'java/', 'go-http', 'postman', 'wget', 'node')): + return '服务器' + return '未知' + + +def contains(address, block): + try: + return ipaddress.ip_address(address) in ipaddress.ip_network(block, strict=False) + except ValueError: + return False + + +def policy_reason(policy, now, device, regions): + """Empty reason means allowed. Disabled / out-of-effect policies do not restrict.""" + if not policy.enabled or (policy.effect_start_time and now < policy.effect_start_time) or ( + policy.effect_end_time and now >= policy.effect_end_time): + return '' + local = now.astimezone(POLICY_TIMEZONE) + if policy.access_time_enabled: + start, end = policy.access_start_time, policy.access_end_time + if start is None or end is None or start == end: + return '访问时段配置无效' + current = local.time().replace(tzinfo=None) + if start < end: + in_period = start <= current < end + day = local.isoweekday() + else: + in_period = current >= start or current < end + day = (local - timedelta(days=1)).isoweekday() if current < end else local.isoweekday() + if not in_period or (policy.allowed_weekdays and day not in policy.allowed_weekdays): + return '不在允许访问时段' + if policy.allowed_device_types and device not in policy.allowed_device_types: + return '设备类型不允许' + if policy.allowed_regions and not set(policy.allowed_regions).intersection(regions): + return '来源区域不允许或无法识别' + return '' + + +def enforce(request, kind, identifier): + if request is None: + return + target = f'{kind}:{identifier}' + checked = getattr(request, '_access_policy_checked', set()) + if target in checked: + return + _, config = configuration() + if not config.get('bindings'): + return + try: + address = client_ip(request) + except AppUnauthorizedFailed: + Log.objects.create(menu='接入控制', operate='拒绝访问', status=403, + operation_object={'target': target}, user={}, ip_address=request.META.get('REMOTE_ADDR', '')[:128], + details={'path': request.path, 'reason': '来源 IP 无效', 'device_type': device_type(request)}) + raise + bindings = [b for b in config.get('bindings', []) if ( + b['target_type'] == 'SERVICE' and b['target_identifier'] == target) or ( + b['target_type'] == 'DEVICE' and contains(address, b['target_identifier']))] + if not bindings: + return + device = device_type(request) + regions = {item['name'] for item in config.get('region_networks', []) if contains(address, item['network'])} + policies = list(AccessControlPolicy.objects.filter(id__in={b['policy_id'] for b in bindings}).order_by('id')) + reason, denied_policy = '', None + now = timezone.now() + for policy in policies: + reason = policy_reason(policy, now, device, regions) + if reason: + denied_policy = policy + break + # DRF has not assigned request.user while an authentication handler runs. + user = request.__dict__.get('_user') + Log.objects.create(menu='接入控制', operate='拒绝访问' if reason else '允许访问', + operation_object={'target': target, 'policy_id': str(denied_policy.id) if denied_policy else ''}, + user={'id': str(user.id)} if getattr(user, 'id', None) else {}, status=403 if reason else 200, + ip_address=address, workspace_id='default', + details={'path': request.path, 'device_type': device, 'regions': sorted(regions), + 'policies': [p.name for p in policies], 'reason': reason}) + if reason: + raise AppUnauthorizedFailed(403, f'接入受限:{reason}') + request._access_policy_checked = checked | {target} + + +def targets(): + from application.models import Application + from system_manage.models.ai_hub import AIService, AIParserResource + return [{'id': f'{kind}:{obj.id}', 'name': obj.name, 'kind': kind, + 'label': f'{label} · {obj.name}'} + for kind, label, model in (('application', '问答助手', Application), + ('mindmap', '导图服务', AIService), ('parser', '解析服务', AIParserResource)) + for obj in model.objects.all().order_by('name')] + + +def enforce_task(request, service_id=None, nodes=None): + """Call before task transactions so denied-call audit survives their rollback.""" + enforce(request, 'mindmap', service_id or 'workbench') + for node in nodes if isinstance(nodes, list) else []: + if isinstance(node, dict) and node.get('type') == 'parse' and node.get('resource_id'): + enforce(request, 'parser', node['resource_id']) + + +@transaction.atomic +def bind(policy_id, operator, data): + policy = AccessControlPolicy.objects.select_for_update().get(pk=policy_id) + kind = data['target_type'] + target = data['target_identifier'].strip() + if kind == 'DEVICE': + target = network(target) + name = data.get('target_name', '').strip() or target + else: + obj = next((item for item in targets() if item['id'] == target), None) + if obj is None: + raise ValidationError('请选择有效服务') + name = obj['label'][:128] + row, config = configuration(lock=True) + item = {'policy_id': str(policy.id), 'target_type': kind, 'target_identifier': target, + 'target_name': name, 'operator_name': operator.nick_name or operator.username, + 'apply_time': timezone.now().isoformat()} + bindings = config.setdefault('bindings', []) + if any(all(b[key] == item[key] for key in ('policy_id', 'target_type', 'target_identifier')) for b in bindings): + raise ValidationError('该对象已绑定此策略') + bindings.append(item) + row.meta = config + row.save(update_fields=['meta', 'update_time']) + AccessControlPolicyApplicationRecord.objects.create(policy=policy, policy_name=policy.name, + target_type=kind, target_identifier=target, target_name=name, + operator=operator, operator_name=item['operator_name'], remark='绑定策略;' + data.get('remark', '')) + policy.application_count += 1 + policy.last_applied_time = timezone.now() + policy.save(update_fields=['application_count', 'last_applied_time', 'update_time']) + return item + + +@transaction.atomic +def unbind(policy_id, operator, data): + policy = AccessControlPolicy.objects.select_for_update().get(pk=policy_id) + row, config = configuration(lock=True) + items = config.get('bindings', []) + removed = [b for b in items if b['policy_id'] == str(policy.id) + and b['target_type'] == data.get('target_type') and b['target_identifier'] == data.get('target_identifier')] + if not removed: + raise ValidationError('绑定不存在') + config['bindings'] = [b for b in items if b not in removed] + row.meta = config + row.save(update_fields=['meta', 'update_time']) + item = removed[0] + AccessControlPolicyApplicationRecord.objects.create(policy=policy, policy_name=policy.name, + target_type=item['target_type'], target_identifier=item['target_identifier'], target_name=item['target_name'], + operator=operator, operator_name=operator.nick_name or operator.username, remark='解除绑定') + + +@transaction.atomic +def save_regions(data): + if not isinstance(data, list) or len(data) > 500: + raise ValidationError('区域网段最多500项') + clean = [] + for item in data: + if not isinstance(item, dict) or not isinstance(item.get('name'), str) or not 1 <= len(item['name'].strip()) <= 64: + raise ValidationError('请输入有效区域名称') + block = network(item.get('network')) + for prior in clean: + left, right = ipaddress.ip_network(prior['network']), ipaddress.ip_network(block) + if left.version == right.version and left.overlaps(right): + raise ValidationError('区域网段不能重叠') + clean.append({'name': item['name'].strip(), 'network': block}) + row, config = configuration(lock=True) + config['region_networks'] = clean + row.meta = config + row.save(update_fields=['meta', 'update_time']) + return clean diff --git a/apps/system_manage/access_control_decisions.py b/apps/system_manage/access_control_decisions.py new file mode 100644 index 000000000..fefdda662 --- /dev/null +++ b/apps/system_manage/access_control_decisions.py @@ -0,0 +1,49 @@ +"""Validated, paginated access decision queries (no configuration mutations).""" +from zoneinfo import ZoneInfo + +from rest_framework import serializers + + +class DecisionQuery(serializers.Serializer): + page = serializers.IntegerField(min_value=1, default=1) + page_size = serializers.IntegerField(min_value=1, max_value=100, default=20) + start_time = serializers.DateTimeField(required=False, default_timezone=ZoneInfo('Asia/Shanghai')) + end_time = serializers.DateTimeField(required=False, default_timezone=ZoneInfo('Asia/Shanghai')) + result = serializers.ChoiceField(choices=['allowed', 'denied'], required=False, allow_blank=True) + ip = serializers.CharField(required=False, allow_blank=True, max_length=128) + device_type = serializers.ChoiceField( + choices=['桌面端', '移动终端', '平板', '服务器', '未知'], required=False, allow_blank=True) + reason = serializers.CharField(required=False, allow_blank=True, max_length=200) + denied = serializers.BooleanField(required=False) + + def validate(self, data): + if data.get('start_time') and data.get('end_time') and data['start_time'] > data['end_time']: + raise serializers.ValidationError({'end_time': '结束时间不能早于开始时间'}) + return data + + +def list_decisions(params): + from system_manage.models.log_management import Log + + serializer = DecisionQuery(data=params) + serializer.is_valid(raise_exception=True) + filters = serializer.validated_data + query = Log.objects.filter(menu='接入控制', operate__in=['允许访问', '拒绝访问']) + outcome = filters.get('result') or ('denied' if filters.get('denied') else '') + if outcome: + query = query.filter(status=403 if outcome == 'denied' else 200) + for key, lookup in [('start_time', 'create_time__gte'), ('end_time', 'create_time__lte'), + ('ip', 'ip_address__icontains'), ('device_type', 'details__device_type'), + ('reason', 'details__reason__icontains')]: + if filters.get(key): + query = query.filter(**{lookup: filters[key]}) + total = query.count() + size = filters['page_size'] + page = min(filters['page'], max(1, (total + size - 1) // size)) + rows = query.order_by('-create_time', '-id')[(page - 1) * size:page * size] + return {'total': total, 'current': page, 'page_size': size, 'records': [ + {'id': str(item.id), 'time': item.create_time, 'status': item.status, + 'ip': item.ip_address, 'target': (item.operation_object or {}).get('target'), + 'device_type': (item.details or {}).get('device_type', '未知'), + 'reason': (item.details or {}).get('reason', '')} + for item in rows]} diff --git a/apps/system_manage/ai_hub.py b/apps/system_manage/ai_hub.py index 5047c9e9f..2fca5c8a9 100644 --- a/apps/system_manage/ai_hub.py +++ b/apps/system_manage/ai_hub.py @@ -1,6 +1,6 @@ """AI hub business operations. Secrets never cross the browser boundary.""" from urllib.parse import urlsplit -from django.db import transaction +from django.db import connection, transaction from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact @@ -12,8 +12,8 @@ KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService, def admin(user): - from common.constants.permission_constants import RoleConstants - return user.role == str(RoleConstants.ADMIN.value) + from system_manage.platform_access import is_admin + return is_admin(user) def require_admin(user): @@ -26,26 +26,53 @@ def visible(query, user): if admin(user): return query if query.model in (AIParserResource, AIService): + from system_manage.platform_access import grants_for, expand_actions + kind = 'PARSER' if query.model == AIParserResource else 'SERVICE' + granted = {g.target for g in grants_for(user).filter(resource_type=kind) if 'VIEW' in expand_actions(g.actions)} ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id) - or str(user.id) in obj.allowed_users] + or str(user.id) in obj.allowed_users or str(obj.id) in granted] return query.filter(id__in=ids) return query.filter(owner_id=user.id) -def one(kind, pk, workspace, user, edit=False): +def one(kind, pk, workspace, user, edit=False, lock=False): from rest_framework.serializers import UUIDField pk = UUIDField().run_validation(pk) model = KINDS.get(kind) if not model: raise NotFound() - obj = visible(model.objects.filter(workspace_id=workspace), user).filter(id=pk).first() + query = model.objects.filter(workspace_id=workspace, id=pk) + if lock: + query = query.select_for_update() + obj = visible(query, user).first() if not obj: raise NotFound('记录不存在或无权访问') if edit and not admin(user) and str(obj.owner_id) != str(user.id): - raise PermissionDenied('无权修改') + from system_manage.platform_access import resource_actions + resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind) + if not resource_type or 'EDIT' not in resource_actions(user, resource_type, obj.id, workspace): + raise PermissionDenied('无权修改') return obj +def require_use(user, obj): + if admin(user) or str(obj.owner_id) == str(user.id) or str(user.id) in obj.allowed_users: + return + from system_manage.platform_access import resource_actions + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + if 'USE' not in resource_actions(user, kind, obj.id, obj.workspace_id): + raise PermissionDenied('未授权使用该资源') + + +def require_resource_action(user, obj, action): + if admin(user) or str(obj.owner_id) == str(user.id): + return + from system_manage.platform_access import resource_actions + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + if action not in resource_actions(user, kind, obj.id, obj.workspace_id): + raise PermissionDenied('无权操作该资源') + + def text(value, label, required=True, limit=MAX_TEXT): if not isinstance(value, str) or len(value) > limit or (required and not value.strip()): raise ValidationError(f'{label}无效') @@ -83,6 +110,8 @@ def authorized_models(workspace, user): grants = WorkspaceUserResourcePermission.objects.filter( workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL) ids = [g.target for g in grants if 'VIEW' in g.permission_list] + from system_manage.platform_access import grants_for, expand_actions + ids.extend(g.target for g in grants_for(user).filter(resource_type='MODEL') if 'USE' in expand_actions(g.actions)) query = query.filter(id__in=ids) return query @@ -96,8 +125,18 @@ def model_for(workspace, user, model_id): return obj -def serialize(obj, detail=False): +def serialize(obj, detail=False, user=None): data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()} + if user is not None: + data['can_delete'] = admin(user) or str(obj.owner_id) == str(user.id) + if user is not None and isinstance(obj, (AIParserResource, AIService)): + from system_manage.platform_access import resource_actions, ACTIONS + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + actions = set(ACTIONS) if admin(user) or str(obj.owner_id) == str(user.id) else resource_actions(user, kind, obj.id, obj.workspace_id) + if str(user.id) in obj.allowed_users: + actions |= {'VIEW', 'USE'} + data['actions'] = sorted(actions) + data['can_delete'] = 'EDIT' in actions if isinstance(obj, AIParserResource): data.update(enabled=obj.enabled, allowed_users=obj.allowed_users, options=obj.options, credential_configured=bool(obj.credential)) @@ -122,7 +161,13 @@ def serialize(obj, detail=False): def parser_save(workspace, user, data, obj=None): - require_admin(user) + if obj is None: + require_admin(user) + else: + if set(data) - {'allowed_users'}: + require_resource_action(user, obj, 'EDIT') + if 'allowed_users' in data: + require_resource_action(user, obj, 'AUTHORIZE') endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/') parsed = urlsplit(endpoint) if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment: @@ -141,17 +186,8 @@ def parser_save(workspace, user, data, obj=None): if 'allowed_users' in data: obj.allowed_users = users(data['allowed_users']) options = data.get('options', obj.options) - if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier'}: - raise ValidationError('解析参数无效') - if options.get('protocol', 'legacy') not in ('legacy', 'v1') or options.get('tier', 'standard') not in ('flash', 'basic', 'standard', 'advanced'): - raise ValidationError('解析接口或档位无效') - if options.get('backend', 'pipeline') not in ('pipeline', 'hybrid-auto-engine', 'vlm-auto-engine'): - raise ValidationError('不支持的解析后端') - if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'): - raise ValidationError('识别语言无效') - if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options): - raise ValidationError('解析参数无效') - obj.options = options + from system_manage.parsers.catalog import validate_options + obj.options = validate_options(options, endpoint) obj.save() return obj @@ -166,10 +202,9 @@ def parser_test(parser): if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password: raise ValidationError('解析服务地址无效') base_path = parsed.path - for suffix in ('/v1', '/api'): - if base_path.endswith(suffix): - base_path = base_path[:-len(suffix)] - break + if parser.options.get('protocol') == 'v1': + if base_path.endswith('/v1'): + base_path = base_path[:-3] url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', '')) headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} started = time.monotonic() @@ -183,13 +218,15 @@ def parser_test(parser): except ValueError as exc: raise ValidationError('解析服务未返回有效接口定义') from exc paths = data.get('paths') if isinstance(data, dict) else None - v1 = parser.options.get('protocol') == 'v1' or parsed.path.endswith('/v1') - suffix = '/tasks' if v1 else '/file_parse' - if not isinstance(paths, dict) or not any(path.endswith(suffix) and 'post' in methods - for path, methods in paths.items() if isinstance(methods, dict)): - raise ValidationError('未识别到对应的解析接口') + v1 = parser.options.get('protocol') == 'v1' + required = ('/v1/uploads', '/v1/parse/jobs') if v1 else ('/file_parse',) + if not isinstance(paths, dict) or not all(any(path.endswith(suffix) and 'post' in methods + for path, methods in paths.items() if isinstance(methods, dict)) for suffix in required): + raise ValidationError('服务接口与所选 MinerU 版本不匹配') + # Existence of the API does not prove that a local GPU/model backend is ready. return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000), - 'message': '解析接口连接正常'} + 'api_version': data.get('info', {}).get('version', ''), 'message': '解析接口连接正常'} + def validate_nodes(workspace, user, nodes): @@ -205,17 +242,31 @@ def validate_nodes(workspace, user, nodes): kind = node['type'] clean = {'type': kind} if kind == 'parse': - parser = one('parsers', node.get('resource_id'), workspace, user) + if node.get('processor') == 'builtin': + result.append({'type': 'parse', 'processor': 'builtin'}) + continue + parser = one('parsers', node.get('resource_id'), workspace, user, lock=connection.in_atomic_block) + require_use(user, parser) if not parser.enabled: raise ValidationError('解析服务已停用') clean['resource_id'] = str(parser.id) if kind == 'organize': clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id) clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000) - temperature = node.get('temperature', 0.2) - if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: - raise ValidationError('温度无效') - clean['temperature'] = temperature + temperature = node.get('temperature') + if temperature is not None: + if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: + raise ValidationError('温度无效') + clean['temperature'] = temperature + # Absent parameters fall back to the per-model settings in model management. + for field, low, high in ( + ('context_tokens', 4096, 262144), ('output_tokens', 256, 32768), + ('safety_tokens', 256, 8192), ('timeout', 30, 600)): + value = node.get(field) + if value is not None: + clean[field] = integer(value, low, high, '模型参数') + if clean.get('output_tokens', 4096) + clean.get('safety_tokens', 1024) >= clean.get('context_tokens', 32768): + raise ValidationError('输出与预留长度不能超过模型上下文') result.append(clean) return result @@ -224,7 +275,7 @@ def validate_nodes(workspace, user, nodes): def flow_create(workspace, user, data): nodes = validate_nodes(workspace, user, data.get('nodes')) # Every save is immutable; published services always reference a fixed row. - previous = one('flows', data['previous_id'], workspace, user, edit=True) if data.get('previous_id') else None + previous = one('flows', data['previous_id'], workspace, user, edit=True, lock=True) if data.get('previous_id') else None return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name'), '名称', limit=128), nodes=nodes, revision=previous.revision + 1 if previous else 1) @@ -233,7 +284,7 @@ def flow_create(workspace, user, data): @transaction.atomic def service_create(workspace, user, data): require_admin(user) - flow = one('flows', data.get('flow_id'), workspace, user) + flow = one('flows', data.get('flow_id'), workspace, user, lock=True) validate_nodes(workspace, user, flow.nodes) return AIService.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name'), '名称', limit=128), flow=flow, @@ -245,6 +296,8 @@ def make_snapshot(workspace, user, nodes): clean = validate_nodes(workspace, user, nodes) for node in clean: if node['type'] == 'parse': + if node.get('processor') == 'builtin': + continue parser = one('parsers', node['resource_id'], workspace, user) node['options'] = dict(parser.options) node['resource_version'] = parser.updated_at.isoformat() @@ -255,7 +308,7 @@ def make_snapshot(workspace, user, nodes): @transaction.atomic -def submit(workspace, user, data, upload=None, service=None): +def submit(workspace, user, data, upload=None, service=None, request=None): from users.models import User current_user = User.objects.select_for_update().get(id=user.id) if not current_user.is_active: @@ -263,12 +316,16 @@ def submit(workspace, user, data, upload=None, service=None): if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10: raise ValidationError('待处理任务已达上限') if service: + service = one('services', service.pk, workspace, user, lock=True) + require_use(user, service) if not service.enabled: raise ValidationError('服务已停用') nodes = service.flow.nodes else: - nodes = one('flows', data['flow_id'], workspace, user).nodes if data.get('flow_id') else data.get('nodes') + nodes = one('flows', data['flow_id'], workspace, user, lock=True).nodes if data.get('flow_id') else data.get('nodes') snapshot = make_snapshot(workspace, user, nodes) + from system_manage.access_control import enforce_task + enforce_task(request, service.id if service else None, snapshot) markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT) if upload and upload.size > MAX_FILE: raise ValidationError('文件不能超过20MB') @@ -303,7 +360,7 @@ def artifact_save(workspace, user, data, obj=None): obj.history = history obj.revision += 1 else: - task = one('tasks', data['task_id'], workspace, user) if data.get('task_id') else None + task = one('tasks', data['task_id'], workspace, user, lock=True) if data.get('task_id') else None if task and task.status != 'SUCCESS': raise ValidationError('任务尚未成功') obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task, @@ -323,40 +380,43 @@ def artifact_save(workspace, user, data, obj=None): return obj +@transaction.atomic +def delete_record(kind, pk, workspace, user): + obj = one(kind, pk, workspace, user, edit=True, lock=True) + if kind == 'parsers': + reference = [{'type': 'parse', 'resource_id': str(obj.id)}] + if AIFlow.objects.filter(workspace_id=workspace, nodes__contains=reference).exists(): + raise ValidationError('解析服务已被流程引用,请先删除相关流程') + if AITask.objects.filter(workspace_id=workspace, status__in=['PENDING', 'RUNNING'], + snapshot__nodes__contains=reference).exists(): + raise ValidationError('解析服务有未完成任务,请稍后删除') + elif kind == 'flows': + if AIService.objects.filter(flow=obj).exists(): + raise ValidationError('流程已被服务引用,请先删除相关服务') + elif kind == 'services': + if AITask.objects.filter(service=obj, status__in=['PENDING', 'RUNNING']).exists(): + raise ValidationError('服务有未完成任务,请稍后删除') + elif kind == 'tasks': + if obj.status in ('PENDING', 'RUNNING'): + raise ValidationError('待处理或运行中的任务不能删除') + elif kind != 'artifacts': + raise ValidationError('资源类型无效') + # Historical task snapshots and saved artifacts remain independent records. + resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind) + if resource_type: + from system_manage.models.platform_access import PlatformGrant + PlatformGrant.objects.filter(workspace_id=workspace, resource_type=resource_type, + target=str(obj.id)).delete() + obj.delete() + + def parse_file(parser, name, content, options=None, progress_callback=None): import requests from common.utils.rsa_util import rsa_long_decrypt opts = options if options is not None else parser.options headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} - if opts.get('protocol') == 'v1' or parser.endpoint.rstrip('/').endswith('/v1'): - from system_manage.ai_hub_parser_v1 import parse - return parse(parser, name, content, opts, headers, progress_callback=progress_callback) - try: - response = requests.post(parser.endpoint + '/file_parse', files={'files': (name, content)}, headers=headers, - data={'backend': opts.get('backend', 'pipeline'), 'lang_list': opts.get('language', 'ch'), - 'parse_method': 'ocr' if opts.get('ocr') else 'auto', 'return_md': 'true', - 'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true', - 'formula_enable': str(opts.get('formula', True)).lower(), - 'table_enable': str(opts.get('table', True)).lower()}, timeout=(10, 300), allow_redirects=False) - except Exception as exc: - raise ValidationError('解析服务连接失败或超时') from exc - if not 200 <= response.status_code < 300: - raise ValidationError('解析服务调用失败') - try: - payload = response.json() - except ValueError as exc: - raise ValidationError('解析服务返回格式无效') from exc - if not isinstance(payload, dict): - raise ValidationError('解析服务返回格式无效') - entries = payload.get('results', {}) - if not isinstance(entries, dict) or len(entries) != 1: - raise ValidationError('解析结果格式无效') - result = next(iter(entries.values())) - if not isinstance(result, dict): - raise ValidationError('解析结果格式无效') - markdown = text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT) - return {'markdown': markdown, 'source': result.get('middle_json', {}), 'parser_id': str(parser.id), - 'images': result.get('images', {})} + from system_manage.parsers import parse + return parse(parser, name, content, opts, headers, progress_callback=progress_callback) def markdown_tree(markdown): diff --git a/apps/system_manage/ai_hub_knowledge.py b/apps/system_manage/ai_hub_knowledge.py index 10458078e..e42e3d5f8 100644 --- a/apps/system_manage/ai_hub_knowledge.py +++ b/apps/system_manage/ai_hub_knowledge.py @@ -29,6 +29,10 @@ def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter parser_id = (knowledge.meta or {}).get('parser_id') if not parser_id: return None + from system_manage.parsers import MINERU_INPUTS, extension + if extension(file.name) not in MINERU_INPUTS: + # Preserve existing knowledge converters, segmentation and attachment ownership. + return None if user is None: raise PermissionDenied('解析服务调用需要用户身份') from system_manage.ai_hub_worker import execute @@ -45,10 +49,34 @@ def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter raise ValidationError(task.error or '解析失败') from django.core.files.uploadedfile import SimpleUploadedFile from common.handle.impl.text.text_split_handle import TextSplitHandle - content = task.result['markdown'].encode('utf-8') + content = knowledge_images(task.result['markdown'], workspace, knowledge_id, user).encode('utf-8') converted = SimpleUploadedFile(file.name + '.md', content) result = TextSplitHandle().handle(converted, patterns, with_filter, limit, lambda _: content, lambda _: None, overlap) result['name'] = file.name return result, {'resource_id': str(parser_id), 'task_id': str(task.id), 'resource_version': task.snapshot['nodes'][0]['resource_version']} + + +def knowledge_images(markdown, workspace, knowledge_id, user): + """Materialize parser images as knowledge attachments, with existing reader permissions and retention.""" + import mimetypes + import uuid + from knowledge.models import File, FileSourceType + from system_manage.ai_hub_assets import ASSET, image_bytes + replacements = {} + for match in ASSET.finditer(markdown): + if match[0] in replacements: + continue + raw, meta = image_bytes(workspace, user, match[2]) + file_id = uuid.uuid5(uuid.NAMESPACE_URL, f'knowledge-parser:{knowledge_id}:{match[2]}') + if not File.objects.filter(pk=file_id).exists(): + suffix = mimetypes.guess_extension(meta['mime']) or '.jpg' + attachment = File(id=file_id, file_name='parsed-image' + suffix, + source_type=FileSourceType.KNOWLEDGE, source_id=str(knowledge_id), + meta={'knowledge_id': str(knowledge_id), 'parser_asset_id': match[2]}) + attachment.save(raw) + replacements[match[0]] = './oss/file/' + str(file_id) + for before, after in replacements.items(): + markdown = markdown.replace(before, after) + return markdown diff --git a/apps/system_manage/ai_hub_organize.py b/apps/system_manage/ai_hub_organize.py new file mode 100644 index 000000000..d3db06b77 --- /dev/null +++ b/apps/system_manage/ai_hub_organize.py @@ -0,0 +1,111 @@ +"""Bounded hierarchical organization; source and completed chunks survive failures.""" +import re +import time + + +def token_bound(text): + # A conservative UTF-8 byte bound avoids assuming one tokenizer for all providers. + return len(text.encode('utf-8')) + + +def split_markdown(text, budget): + if budget < 256: + raise ValueError('模型上下文预算不足,请调大上下文或减少输出长度') + chunks, current = [], '' + blocks = re.split(r'(?m)(?=^#{1,6} )', text) + for block in blocks: + while block: + if token_bound(current + block) <= budget: + current += block + break + if current: + chunks.append(current); current = '' + if token_bound(block) <= budget: + current = block + break + lo, hi = 1, min(len(block), budget) + while lo < hi: + mid = (lo + hi + 1) // 2 + if token_bound(block[:mid]) <= budget: + lo = mid + else: + hi = mid - 1 + end = lo + newline = block.rfind('\n', 0, end) + if newline > end // 2: + end = newline + 1 + # Never split an image placeholder between requests. + for match in re.finditer(r'AIHUBIMAGE\s*\d+\s*TOKEN', block[:end + 64]): + if match.start() < end < match.end(): + end = match.start() + break + if not end: + raise ValueError('图片占位符超过模型输入预算') + chunks.append(block[:end]); block = block[end:] + if current: + chunks.append(current) + return chunks + + +def organize(markdown, prompt, invoke, *, context_tokens=32768, output_tokens=4096, + safety_tokens=1024, progress=None, checkpoint=None): + budget = context_tokens - output_tokens - safety_tokens - token_bound(prompt) - 512 + chunks = split_markdown(markdown, budget) + if len(chunks) > 128: + raise ValueError('文档分块过多,请调大模型上下文或拆分文档') + calls = 0 + + def call(content, instruction): + nonlocal calls + calls += 1 + if calls > 160: + raise ValueError('整理批次过多,请拆分文档') + if token_bound(content) + token_bound(instruction) > context_tokens - output_tokens - safety_tokens: + raise ValueError('模型输入超过上下文预算') + for attempt in range(3): + try: + response = invoke(instruction, content) + break + except Exception as exc: + if attempt == 2: + raise ValueError('模型调用失败,已保留原稿和完成的分块,请稍后重试') from exc + time.sleep(attempt + 1) + metadata = getattr(response, 'response_metadata', {}) + if isinstance(metadata, dict) and metadata.get('finish_reason') in ('length', 'max_tokens'): + raise ValueError('模型输出被截断,请到模型管理调大该模型的输出上限,或减少整理内容') + result = getattr(response, 'content', response) + if isinstance(result, list): + result = ''.join(item.get('text', '') for item in result if isinstance(item, dict)) + if not isinstance(result, str) or not result.strip(): + raise ValueError('模型未返回有效整理内容') + return result.strip() + + outputs = [] + chunk_prompt = prompt + '\n这是文档的一个分块,保留事实及标题关系,精简重复内容。' + for index, chunk in enumerate(chunks, 1): + if progress: + progress({'stage': f'智能整理分块 {index}/{len(chunks)}', 'percent': (index - 1) * 70 // len(chunks)}) + outputs.append(call(chunk, prompt if len(chunks) == 1 else chunk_prompt)) + if checkpoint: + checkpoint(list(outputs)) + if len(outputs) == 1: + return outputs[0] + merge_prompt = prompt + '\n合并以下局部大纲,统一标题层级,压缩重复内容;保留事实、数字和图片占位符。' + previous_size = None + for round_index in range(4): + joined = '\n\n'.join(outputs) + groups = split_markdown(joined, budget) + if previous_size is not None and token_bound(joined) >= previous_size and len(groups) > 1: + raise ValueError('合并结果未收敛,已保留完成的分块,请缩小文档或调整提示词') + previous_size = token_bound(joined) + merged = [] + for index, group in enumerate(groups, 1): + if progress: + progress({'stage': f'合并大纲 {round_index + 1}:{index}/{len(groups)}', 'percent': 70 + round_index * 6}) + merged.append(call(group, merge_prompt)) + if checkpoint: + checkpoint(merged) + if len(merged) == 1: + return merged[0] + outputs = merged + raise ValueError('合并批次超过限制,已保留完成的分块') diff --git a/apps/system_manage/ai_hub_parser_v1.py b/apps/system_manage/ai_hub_parser_v1.py index 4ee7128fc..ad8faa9ea 100644 --- a/apps/system_manage/ai_hub_parser_v1.py +++ b/apps/system_manage/ai_hub_parser_v1.py @@ -1,59 +1,165 @@ -"""MinerU v1 task API adapter. Only follows paths built from the registered endpoint.""" +"""MinerU 4 V1 upload/job/file protocol, independent of convenience routes.""" +import base64 +import hashlib +import io +import json +import mimetypes import re import time +import zipfile +from pathlib import PurePosixPath +from urllib.parse import urljoin, urlsplit import requests from rest_framework.exceptions import ValidationError +MAX_OUTPUT = 150 * 1024 * 1024 -def _json(response): - if response.status_code not in (200, 201, 202): - raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + +def origin(url): + p = urlsplit(url) + if p.scheme not in ('http', 'https') or not p.hostname or p.username or p.password: + raise ValidationError('解析服务返回地址无效') + return p.scheme.lower(), p.hostname.lower(), p.port or (443 if p.scheme == 'https' else 80) + + +def identifier(value): + if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,200}', value): + raise ValidationError('解析服务返回资源 ID 无效') + return value + + +def read(response): + chunks, size = [], 0 try: - body = response.json() - except ValueError as exc: + for chunk in response.iter_content(65536): + size += len(chunk) + if size > MAX_OUTPUT: + raise ValidationError('解析结果超过大小限制') + chunks.append(chunk) + finally: + response.close() + return b''.join(chunks) + + +def call(method, url, headers, expect_json=True, **kwargs): + response = requests.request(method, url, headers=headers, timeout=(10, 30), allow_redirects=False, + stream=True, **kwargs) + if response.status_code not in (200, 201, 202, 204): + response.close() + raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + raw = read(response) + if not expect_json or response.status_code == 204 or not raw: + return {} + try: + result = json.loads(raw) + except (ValueError, UnicodeDecodeError) as exc: raise ValidationError('解析服务返回格式无效') from exc - if not isinstance(body, dict): + if not isinstance(result, dict): raise ValidationError('解析服务返回格式无效') - return body + return result + + +def download(url, base, headers): + for _ in range(4): + response = requests.get(url, headers=headers if origin(url) == origin(base) else {}, + timeout=(10, 30), allow_redirects=False, stream=True) + if response.status_code in (301, 302, 303, 307, 308): + target = response.headers.get('Location', '') + response.close() + if not target: + raise ValidationError('解析结果跳转地址无效') + url = urljoin(url, target) + origin(url) + continue + if response.status_code != 200: + response.close() + raise ValidationError(f'获取解析结果失败(HTTP {response.status_code})') + return read(response) + raise ValidationError('解析结果跳转次数过多') + + +def unpack(raw): + images = {} + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + if sum(i.file_size for i in archive.infolist()) > 200 * 1024 * 1024: + raise ValidationError('解析结果解压后超过大小限制') + for item in archive.infolist(): + path = PurePosixPath(item.filename) + if '..' in path.parts or path.is_absolute(): + raise ValidationError('解析结果路径无效') + mime = mimetypes.guess_type(item.filename)[0] or '' + if mime.startswith('image/'): + images[item.filename] = 'data:' + mime + ';base64,' + base64.b64encode(archive.read(item)).decode() + except (zipfile.BadZipFile, RuntimeError) as exc: + raise ValidationError('解析结果压缩包无效') from exc + return images def parse(parser, name, content, options, headers, progress_callback=None): from system_manage.ai_hub import text, MAX_INLINE_TEXT + from system_manage.ai_hub_progress import parser_progress endpoint = parser.endpoint.rstrip('/') + base = endpoint if endpoint.endswith('/v1') else endpoint + '/v1' + root = base[:-3] deadline = time.monotonic() + 300 try: - response = requests.post(endpoint + '/tasks', files={'files': (name, content)}, headers=headers, - params=[('tier', options.get('tier', 'standard')), ('ocr_mode', 'ocr' if options.get('ocr') else 'auto'), - ('output_formats', 'markdown'), ('output_formats', 'middle_json')], - timeout=(10, 30), allow_redirects=False) - task = _json(response) - from system_manage.ai_hub_progress import parser_progress - task_id = task.get('task_id', '') - if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id): - raise ValidationError('解析服务未返回有效任务ID') - if progress_callback: - progress_callback(parser_progress(task)) - while task.get('status') in ('queued', 'running'): - if time.monotonic() >= deadline: - raise ValidationError('解析等待超时,后台任务ID:' + task_id) - time.sleep(2) - task = _json(requests.get(endpoint + '/tasks/' + task_id, headers=headers, - timeout=(10, 20), allow_redirects=False)) + upload = call('POST', base + '/uploads', headers, json={ + 'filename': name, 'bytes': len(content), 'mime_type': mimetypes.guess_type(name)[0] or 'application/octet-stream', + 'purpose': 'parse', 'sha256sum': hashlib.sha256(content).hexdigest()}) + if upload.get('status') == 'pending': + uid = identifier(upload.get('id')) + target = urljoin(root + '/', upload.get('upload_url', '')) + if not upload.get('upload_url') or upload.get('upload_method', 'PUT') != 'PUT': + raise ValidationError('解析上传协议无效') + origin(target) + extra = upload.get('upload_headers') or {} + if not isinstance(extra, dict) or any(not isinstance(k, str) or not isinstance(v, str) or '\n' in k+v or '\r' in k+v for k,v in extra.items()): + raise ValidationError('解析上传请求头无效') + # Never forward the registered parser credential to object storage. + auth = {**extra, **headers} if origin(target) == origin(base) else extra + call('PUT', target, auth, expect_json=False, data=content) + upload = call('POST', base + '/uploads/' + uid + '/complete', headers) + if upload.get('status') != 'completed': + raise ValidationError('解析文件上传未完成') + fid = identifier((upload.get('file') or {}).get('id')) + job = call('POST', base + '/parse/jobs', headers, json={ + 'files': [{'source': {'type': 'file_id', 'file_id': fid}}], + 'tier': options.get('tier', 'standard'), 'ocr_mode': 'ocr' if options.get('ocr') else 'auto', + 'output_formats': ['markdown', 'middle_json', 'zip']}) + jid = identifier(job.get('job_id')) + while True: + state = job.get('status') + if state not in ('queued', 'running', 'completed', 'partial', 'failed', 'canceled'): + raise ValidationError('解析任务状态无效') if progress_callback: - progress_callback(parser_progress(task)) - if task.get('status') != 'completed': - raise ValidationError('解析任务未成功,后台任务ID:' + task_id) - result = _json(requests.get(endpoint + '/tasks/' + task_id + '/result', headers=headers, - timeout=(10, 30), allow_redirects=False)) + progress_callback(parser_progress(job)) + if state not in ('queued', 'running'): + break + if time.monotonic() >= deadline: + raise ValidationError('解析等待超时,后台任务 ID:' + jid) + time.sleep(2) + job = call('GET', base + '/parse/jobs/' + jid, headers) + files = job.get('files') + if state != 'completed' or not isinstance(files, list) or len(files) != 1 or files[0].get('status') != 'completed': + raise ValidationError('解析任务未成功,后台任务 ID:' + jid) + outputs = files[0].get('output_files') or {} + def artifact(kind): + ref = outputs.get(kind) + if not isinstance(ref, dict): + raise ValidationError('解析结果缺少 ' + kind) + return download(base + '/files/' + identifier(ref.get('file_id')) + '/content', base, headers) + markdown = artifact('markdown').decode('utf-8') + middle = json.loads(artifact('middle_json')) + images = unpack(artifact('zip')) if 'zip' in outputs else {} + # ZIP outputs can wrap a document directory; include relative image paths. + for path, value in list(images.items()): + parts = PurePosixPath(path).parts + if 'images' in parts: + images['/'.join(parts[parts.index('images'):])] = value + return {'markdown': text(markdown, '解析结果', limit=MAX_INLINE_TEXT), 'parser_id': str(parser.id), + 'images': images, 'source': {'task_id': jid, 'middle_json': middle}} except requests.RequestException as exc: raise ValidationError('解析服务连接失败或超时') from exc - files = result.get('files', []) - if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) or files[0].get('status') != 'completed': - raise ValidationError('解析文件结果无效') - output = files[0].get('content', {}) - if not isinstance(output, dict): - raise ValidationError('解析文件内容无效') - markdown = text(output.get('markdown'), '解析结果', limit=MAX_INLINE_TEXT) - return {'markdown': markdown, 'parser_id': str(parser.id), 'images': output.get('images', {}), - 'source': {'task_id': task_id, 'parse': files[0].get('parse'), - 'middle_json': output.get('middle_json', {})}} + except (ValueError, UnicodeDecodeError) as exc: + raise ValidationError('解析结果内容无效') from exc diff --git a/apps/system_manage/ai_hub_worker.py b/apps/system_manage/ai_hub_worker.py index eaff5de81..ce1f5cf7c 100644 --- a/apps/system_manage/ai_hub_worker.py +++ b/apps/system_manage/ai_hub_worker.py @@ -23,8 +23,12 @@ def execute(task): user = User.objects.get(pk=task.owner_id) if not user.is_active: raise ValueError('用户已停用') + from system_manage.platform_access import module_list + if 'mindmap' not in module_list(user): + raise ValueError('未授权使用思维导图') if task.service_id: service = hub.one('services', task.service_id, task.workspace_id, user) + hub.require_use(user, service) if not service.enabled: raise ValueError('服务已停用') markdown = task.input_data.get('markdown', '') @@ -39,15 +43,23 @@ def execute(task): task.steps = steps task.save(update_fields=['steps', 'updated_at']) if node['type'] == 'parse': - parser = hub.one('parsers', node['resource_id'], task.workspace_id, user) - if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']: - raise ValueError('解析服务已停用或版本发生变化,请重新提交') + parser = None + if node.get('processor') != 'builtin': + parser = hub.one('parsers', node['resource_id'], task.workspace_id, user) + hub.require_use(user, parser) + if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']: + raise ValueError('解析服务已停用或版本发生变化,请重新提交') def report_progress(progress): step['progress'] = progress AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update( steps=steps, updated_at=timezone.now()) report_progress({'stage': '上传文档到解析服务', 'percent': None}) - parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) + if parser is None: + from system_manage.parsers.native import parse + report_progress({'stage': '使用内置处理器解析文档', 'percent': None}) + parsed = parse(task.file_name, bytes(task.file_data)) + else: + parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) from system_manage.ai_hub_assets import asset_lock, normalize with asset_lock(): markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果') @@ -66,17 +78,33 @@ def execute(task): # Bound input; never silently truncate the original document. from system_manage.ai_hub_assets import protect_images, restore_images model_markdown, image_tokens = protect_images(markdown) - if len(model_markdown) > 60000: - raise ValueError('文档超过智能整理长度限制,请分批处理') try: - instance = get_model(model, **{**get_model_default_params(model), - 'temperature': node.get('temperature', 0.2), 'timeout': 180}) + # Per-model settings from model management win; task nodes override only when set. + defaults = get_model_default_params(model) + temperature = node.get('temperature') + if temperature is None: + temperature = defaults.get('temperature', 0.2) + max_tokens = node.get('output_tokens') + if max_tokens is None: + max_tokens = defaults.get('max_tokens', 4096) + instance = get_model(model, **{**defaults, 'temperature': temperature, + 'max_tokens': max_tokens, 'timeout': node.get('timeout', 180)}) prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。' - response = instance.invoke([SystemMessage(content=prompt), HumanMessage(content=model_markdown)]) except Exception as exc: from rest_framework.exceptions import ValidationError raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc - markdown = restore_images(hub.text(response.content, '整理结果'), image_tokens) + from system_manage.ai_hub_organize import organize + def organize_progress(progress): + step['progress'] = progress + AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now()) + def checkpoint(outputs): + task.result['organize_chunks'] = outputs + task.save(update_fields=['result', 'updated_at']) + organized = organize(model_markdown, prompt, + lambda instruction, content: instance.invoke([SystemMessage(content=instruction), HumanMessage(content=content)]), + context_tokens=node.get('context_tokens', 32768), output_tokens=max_tokens, + safety_tokens=node.get('safety_tokens', 1024), progress=organize_progress, checkpoint=checkpoint) + markdown = restore_images(hub.text(organized, '整理结果'), image_tokens) else: tree = hub.markdown_tree(markdown) step['status'] = 'SUCCESS' diff --git a/apps/system_manage/document_export.py b/apps/system_manage/document_export.py new file mode 100644 index 000000000..073155bef --- /dev/null +++ b/apps/system_manage/document_export.py @@ -0,0 +1,305 @@ +"""Structured DOCX authoring and bounded, isolated LibreOffice conversions. + +No URL fetching, shell interpolation, macros, shared profiles, or database writes. +""" +import base64 +import contextlib +import io +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import time + +from bs4 import BeautifulSoup, NavigableString, Tag +from docx import Document +from docx.enum.text import WD_BREAK +from docx.oxml import OxmlElement +from docx.oxml.ns import qn +from docx.shared import Inches, Pt +from PIL import Image + +MAX_HTML = 24 * 1024 * 1024 +MAX_IMAGES = 16 * 1024 * 1024 +MIMES = {'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'pdf': 'application/pdf', 'rtf': 'application/rtf'} + + +class ExportError(ValueError): + pass + + +def _element(parent, name, **attrs): + child = OxmlElement(name) + for key, value in attrs.items(): + child.set(qn('w:' + key), str(value)) + parent.append(child) + return child + + +@contextlib.contextmanager +def conversion_slot(): + # A machine-wide file lock also bounds work across multiple Web processes. + path = Path(tempfile.gettempdir()) / 'maxkb-document-export.lock' + with path.open('a+b') as stream: + stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0) + deadline = time.monotonic() + 15 + while True: + try: + if os.name == 'nt': + import msvcrt + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except OSError: + if time.monotonic() > deadline: + raise ExportError('导出繁忙,请稍后重试') + time.sleep(.1) + try: + yield + finally: + if os.name == 'nt': + import msvcrt + stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + + +class DocumentBuilder: + def __init__(self, image_resolver=None, paper='A4'): + self.doc = Document() + self.resolve = image_resolver + self.image_bytes = 0 + self.image_count = 0 + section = self.doc.sections[0] + section.page_width = Inches(8.2677) + section.page_height = Inches(11.6929) + section.top_margin = section.bottom_margin = Inches(.75) + section.left_margin = section.right_margin = Inches(.8) + self.width = section.page_width - section.left_margin - section.right_margin + for name in ['Normal', 'Title', *['Heading ' + str(i) for i in range(1, 7)]]: + style = self.doc.styles[name] + style.font.name = 'Noto Sans CJK SC' + style.font.size = Pt(11 if name == 'Normal' else 22 if name == 'Title' else max(11, 19 - int(name[-1]))) + style.font.color.rgb = __import__('docx').shared.RGBColor(0, 0, 0) + fonts = style.element.get_or_add_rPr().rFonts + fonts.set(qn('w:eastAsia'), 'Noto Sans CJK SC') + style.paragraph_format.space_after = Pt(6) + style.paragraph_format.line_spacing = 1.25 + style.paragraph_format.widow_control = True + footer = section.footer.paragraphs[0] + footer.alignment = 1 + field = _element(footer._p, 'w:fldSimple', instr='PAGE') + _element(_element(field, 'w:r'), 'w:t').text = '1' + + def image(self, node, paragraph, width=None): + self.image_count += 1 + if self.image_count > 128: raise ExportError('单次导出图片不能超过128张') + src = node.get('src', '') + if src.startswith('data:image/'): + match = re.fullmatch(r'data:image/(png|jpeg|gif|webp|bmp);base64,([A-Za-z0-9+/=\s]+)', src) + if not match: + raise ExportError('图片格式无效') + try: + raw = base64.b64decode(match[2], validate=False) + except (ValueError, TypeError) as exc: + raise ExportError('图片格式无效') from exc + elif self.resolve: + raw = self.resolve(src) + else: + raise ExportError('图片未保存,请重新加载后导出') + self.image_bytes += len(raw) + if self.image_bytes > MAX_IMAGES: + raise ExportError('导出图片总大小不能超过16MB') + try: + with Image.open(io.BytesIO(raw)) as image: + if image.width * image.height > 40_000_000: + raise ExportError('图片尺寸过大') + image.load() + # DOCX/RTF and Office readers share a reliable raster baseline. + converted = io.BytesIO() + image.convert('RGBA' if image.mode == 'RGBA' else 'RGB').save(converted, format='PNG') + natural_width, natural_height = image.size + max_width = min(width or self.width, self.width) + requested_width = str(node.get('width', natural_width)) + pixels = int(requested_width) if requested_width.isdigit() and 0 < int(requested_width) < 10000 else natural_width + display_width = min(max_width, Inches(pixels / 96)) + display_width = min(display_width, Inches(8.8) * natural_width / max(1, natural_height)) + picture = paragraph.add_run().add_picture(io.BytesIO(converted.getvalue()), width=int(display_width)) + picture._inline.docPr.set('descr', node.get('alt', '')) + except ExportError: + raise + except Exception as exc: + raise ExportError('图片无法导出,请检查图片格式') from exc + + def inline(self, node, paragraph, formats=None, width=None): + formats = dict(formats or {}) + if isinstance(node, NavigableString): + run = paragraph.add_run(str(node)) + for name in ('bold', 'italic', 'underline', 'strike'): + if formats.get(name): + if name == 'strike': run.font.strike = True + else: setattr(run, name, True) + if formats.get('subscript'): run.font.subscript = True + if formats.get('superscript'): run.font.superscript = True + if formats.get('color'): + from docx.shared import RGBColor + run.font.color.rgb = RGBColor.from_string(formats['color']) + if formats.get('code'): + run.font.name = 'DejaVu Sans Mono'; run.font.size = Pt(9) + return + if not isinstance(node, Tag): + return + tag = node.name.lower() + if tag in ('script', 'style', 'iframe', 'object', 'svg', 'math'): + return + if tag == 'img': + self.image(node, paragraph, width); return + if tag == 'br': + paragraph.add_run().add_break(WD_BREAK.LINE); return + mapping = {'b': 'bold', 'strong': 'bold', 'i': 'italic', 'em': 'italic', 'u': 'underline', 's': 'strike', 'del': 'strike', 'code': 'code', 'sub': 'subscript', 'sup': 'superscript'} + if tag in mapping: formats[mapping[tag]] = True + color = re.search(r'(?:^|;)\s*color\s*:\s*#([0-9a-fA-F]{6})(?:;|$)', node.get('style', '')) + if color: formats['color'] = color[1] + if tag == 'a': + href = node.get('href', '') + if re.match(r'^(https?://|mailto:)', href, re.I): + from docx.opc.constants import RELATIONSHIP_TYPE as RT + hyperlink = _element(paragraph._p, 'w:hyperlink') + hyperlink.set(qn('r:id'), paragraph.part.relate_to(href, RT.HYPERLINK, is_external=True)) + # Build styled text with normal runs, then move them into the link. + start = len(paragraph.runs) + for child in node.children: self.inline(child, paragraph, formats, width) + for run in list(paragraph.runs)[start:]: hyperlink.append(run._r) + return + for child in node.children: + self.inline(child, paragraph, formats, width) + + def numbering(self, ordered, start=1): + numbering = self.doc.part.numbering_part.element + abstract_id = max([int(n.get(qn('w:abstractNumId'))) for n in numbering.findall(qn('w:abstractNum'))] + [-1]) + 1 + abstract = _element(numbering, 'w:abstractNum', abstractNumId=abstract_id) + _element(abstract, 'w:multiLevelType', val='multilevel') + for level in range(9): + lvl = _element(abstract, 'w:lvl', ilvl=level) + _element(lvl, 'w:start', val=start if level == 0 else 1) + _element(lvl, 'w:numFmt', val='decimal' if ordered else 'bullet') + _element(lvl, 'w:lvlText', val=f'%{level + 1}.' if ordered else '•') + _element(_element(lvl, 'w:pPr'), 'w:ind', left=360 * (level + 1), hanging=240) + num_id = max([int(n.get(qn('w:numId'))) for n in numbering.findall(qn('w:num'))] + [0]) + 1 + _element(_element(numbering, 'w:num', numId=num_id), 'w:abstractNumId', val=abstract_id) + return num_id + + def blocks(self, nodes, container=None, level=0, width=None): + container = self.doc if container is None else container + width = width or self.width + for node in nodes: + if isinstance(node, NavigableString): + if str(node).strip(): container.add_paragraph(str(node)) + continue + if not isinstance(node, Tag): continue + tag = node.name.lower() + if tag in ('script', 'style', 'iframe', 'object'): continue + if tag in ('ul', 'ol'): + num_id = self.numbering(tag == 'ol', min(100000, max(1, int(node.get('start', 1))))) + for item in node.find_all('li', recursive=False): + p = container.add_paragraph() + props = _element(p._p.get_or_add_pPr(), 'w:numPr') + _element(props, 'w:ilvl', val=min(level, 8)); _element(props, 'w:numId', val=num_id) + for child in item.children: + if isinstance(child, Tag) and child.name in ('ul', 'ol'): + self.blocks([child], container, level + 1, width) + else: self.inline(child, p, width=width) + continue + if tag == 'table': + rows = node.find_all('tr') + if len(rows) > 2000: raise ExportError('表格行数过多') + occupied = set(); placements = []; columns = 0 + for row_index, row in enumerate(rows): + column = 0 + for cell in row.find_all(['td', 'th'], recursive=False): + while (row_index, column) in occupied: column += 1 + colspan = min(30, max(1, int(cell.get('colspan', 1)))) + rowspan = min(len(rows) - row_index, max(1, int(cell.get('rowspan', 1)))) + if column + colspan > 30: raise ExportError('表格列数不能超过30列') + for rr in range(row_index, row_index + rowspan): + for cc in range(column, column + colspan): + if (rr, cc) in occupied: raise ExportError('表格合并区域无效') + occupied.add((rr, cc)) + placements.append((row_index, column, rowspan, colspan, cell)) + column += colspan; columns = max(columns, column) + if not columns: continue + table = container.add_table(rows=len(rows), cols=columns) + table.style = 'Table Grid'; table.autofit = False + for column in table.columns: column.width = int(width / columns) + table_rows = list(table.rows) + # python-docx reconstructs the entire grid on every table.cell() + # call. Cache it once; placement targets never point inside a merged span. + grid = list(table._cells) + for index, row in enumerate(rows): + if row.find('th') is not None: _element(table_rows[index]._tr.get_or_add_trPr(), 'w:tblHeader') + _element(table_rows[index]._tr.get_or_add_trPr(), 'w:cantSplit') + for rr, cc, rowspan, colspan, cell in placements: + target = grid[rr * columns + cc] + if rowspan > 1 or colspan > 1: target = target.merge(grid[(rr + rowspan - 1) * columns + cc + colspan - 1]) + p = target.paragraphs[0] + alignment = re.search(r'text-align:\s*(left|center|right)', cell.get('style', '')) + if alignment: p.alignment = {'left':0, 'center':1, 'right':2}[alignment[1]] + for child in cell.children: self.inline(child, p, {'bold': cell.name == 'th'}, int(width / columns * colspan)) + container.add_paragraph(); continue + if tag in ('div', 'section', 'article', 'main', 'body', 'blockquote'): + before = len(container.paragraphs) + self.blocks(list(node.children), container, level, width) + if tag == 'blockquote': + for p in container.paragraphs[before:]: p.paragraph_format.left_indent = Inches(.25) + continue + if tag in ('thead', 'tbody', 'tfoot'): continue + p = container.add_paragraph() + if re.fullmatch('h[1-6]', tag): + p.style = 'Heading ' + tag[1]; p.paragraph_format.keep_with_next = True + if tag == 'pre': + p.paragraph_format.space_before = Pt(5) + shading = _element(p._p.get_or_add_pPr(), 'w:shd', fill='F5F7FA') + run = p.add_run(node.get_text()); run.font.name = 'DejaVu Sans Mono'; run.font.size = Pt(9) + elif tag == 'hr': + _element(_element(p._p.get_or_add_pPr(), 'w:pBdr'), 'w:bottom', val='single', sz=4, color='D9DDE4') + else: self.inline(node, p, width=width) + + def build(self, html): + soup = BeautifulSoup(html, 'html.parser') + if len(soup.find_all(True)) > 50000: raise ExportError('文档结构过于复杂,请拆分后导出') + self.blocks(list((soup.body or soup).children)) + result = io.BytesIO(); self.doc.save(result) + return result.getvalue() + + +def export_document(html, format_name='docx', image_resolver=None, paper='A4'): + if format_name not in MIMES: raise ExportError('不支持的导出格式') + if paper != 'A4': raise ExportError('仅支持 A4 纸张') + if not isinstance(html, str) or not html.strip() or len(html.encode('utf-8')) > MAX_HTML: + raise ExportError('导出内容为空或超过24MB') + raw = DocumentBuilder(image_resolver, paper).build(html) + if format_name == 'docx': return raw, MIMES[format_name] + command = shutil.which('libreoffice') or shutil.which('soffice') + if command is None: raise ExportError('文档转换组件不可用,请联系管理员或下载 DOCX') + with conversion_slot(), tempfile.TemporaryDirectory(prefix='maxkb-export-') as directory: + root = Path(directory) + (root / 'document.docx').write_bytes(raw) + try: + completed = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(), + '--headless', '--convert-to', format_name, '--outdir', str(root), str(root / 'document.docx')], + stdin=subprocess.DEVNULL, capture_output=True, timeout=90, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ExportError('文档转换超时或失败,请下载 DOCX 后重试') from exc + result = root / ('document.' + format_name) + if completed.returncode or not result.is_file() or result.stat().st_size == 0: + raise ExportError('文档转换失败,请下载 DOCX 后重试') + output = result.read_bytes() + if format_name == 'pdf' and not output.startswith(b'%PDF-'): raise ExportError('PDF 输出无效') + if format_name == 'rtf' and not output.lstrip().startswith(b'{\\rtf'): raise ExportError('RTF 输出无效') + return output, MIMES[format_name] diff --git a/apps/system_manage/management/commands/unify_platform_roles.py b/apps/system_manage/management/commands/unify_platform_roles.py new file mode 100644 index 000000000..4981de4ed --- /dev/null +++ b/apps/system_manage/management/commands/unify_platform_roles.py @@ -0,0 +1,11 @@ +"""Import existing account roles into the unified role/member tables.""" +import json +from django.core.management.base import BaseCommand +from system_manage.platform_access import reconcile_builtin_roles + + +class Command(BaseCommand): + help = 'Backfill built-in platform roles; preserves custom roles and resource grants. No schema migration.' + + def handle(self, *args, **options): + self.stdout.write(json.dumps(reconcile_builtin_roles(), ensure_ascii=False)) diff --git a/apps/system_manage/migrations/0008_platform_access.py b/apps/system_manage/migrations/0008_platform_access.py new file mode 100644 index 000000000..749589e94 --- /dev/null +++ b/apps/system_manage/migrations/0008_platform_access.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.8 on 2026-10-23 06:05 + +import django.contrib.postgres.fields +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +EFFECTIVE_VIEW = """ +CREATE VIEW effective_resource_permission AS +SELECT id, workspace_id, user_id, auth_target_type, target, auth_type, + permission_list, create_time, update_time +FROM workspace_user_resource_permission +UNION ALL +SELECT g.id, g.workspace_id, u.id, g.resource_type::varchar(128), g.target, + 'RESOURCE_PERMISSION_GROUP'::varchar, + (ARRAY['VIEW']::varchar[] || CASE WHEN g.actions ? 'EDIT' + THEN ARRAY['MANAGE']::varchar[] ELSE ARRAY[]::varchar[] END), + g.updated_at, g.updated_at +FROM platform_grant g JOIN "user" u ON u.is_active = true +WHERE g.resource_type IN ('KNOWLEDGE','APPLICATION','MODEL','TOOL') + AND g.actions <> '[]'::jsonb + AND (g.expires_at IS NULL OR g.expires_at > statement_timestamp()) + AND NOT EXISTS (SELECT 1 FROM user_activation_application a WHERE a.user_id=u.id AND a.status <> 'APPROVED') + AND (g.subject_type='ALL' + OR (g.subject_type='USER' AND g.subject_id=u.id::text) + OR (g.subject_type='ROLE' AND EXISTS ( + SELECT 1 FROM platform_role r WHERE r.id::text=g.subject_id AND r.enabled + AND (r.is_default OR EXISTS (SELECT 1 FROM platform_role_member m WHERE m.role_id=r.id AND m.user_id=u.id))))) +""" + + +def seed_default(apps, schema_editor): + apps.get_model('system_manage', 'PlatformRole').objects.get_or_create( + id='00000000-0000-0000-0000-000000000001', + defaults={'name': '普通用户', 'description': '所有已激活用户自动关联', + 'is_default': True, 'modules': ['knowledge', 'mindmap']}) + + +class Migration(migrations.Migration): + + dependencies = [ + ('system_manage', '0007_ai_hub'), + ('users', '0002_user_activation_and_role_assignment'), + ] + + operations = [ + migrations.CreateModel( + name='EffectiveResourcePermission', + fields=[ + ('id', models.UUIDField(primary_key=True, serialize=False)), + ('workspace_id', models.CharField(max_length=128)), + ('user_id', models.UUIDField()), + ('auth_target_type', models.CharField(max_length=128)), + ('target', models.CharField(max_length=128)), + ('auth_type', models.CharField(max_length=64)), + ('permission_list', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), size=None)), + ('create_time', models.DateTimeField()), + ('update_time', models.DateTimeField()), + ], + options={ + 'db_table': 'effective_resource_permission', + 'managed': False, + }, + ), + migrations.CreateModel( + name='PlatformRole', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=64, unique=True)), + ('description', models.CharField(blank=True, default='', max_length=256)), + ('enabled', models.BooleanField(default=True)), + ('is_default', models.BooleanField(default=False)), + ('modules', models.JSONField(default=list)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'platform_role', + }, + ), + migrations.CreateModel( + name='PlatformGrant', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('subject_type', models.CharField(choices=[('ALL', '所有用户'), ('ROLE', '角色'), ('USER', '用户')], max_length=8)), + ('subject_id', models.CharField(blank=True, default='', max_length=64)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('resource_type', models.CharField(max_length=32)), + ('target', models.CharField(max_length=128)), + ('actions', models.JSONField(default=list)), + ('expires_at', models.DateTimeField(blank=True, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('granted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user')), + ], + options={ + 'db_table': 'platform_grant', + 'indexes': [models.Index(fields=['subject_type', 'subject_id'], name='platform_grant_subject_idx')], + 'constraints': [models.UniqueConstraint(fields=('workspace_id', 'subject_type', 'subject_id', 'resource_type', 'target'), name='unique_platform_grant')], + }, + ), + migrations.CreateModel( + name='PlatformRoleMember', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='system_manage.platformrole')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='platform_roles', to='users.user')), + ], + options={ + 'db_table': 'platform_role_member', + 'constraints': [models.UniqueConstraint(fields=('role', 'user'), name='unique_platform_role_member')], + }, + ), + migrations.RunSQL(EFFECTIVE_VIEW, 'DROP VIEW IF EXISTS effective_resource_permission'), + migrations.RunPython(seed_default, migrations.RunPython.noop), + ] diff --git a/apps/system_manage/models/__init__.py b/apps/system_manage/models/__init__.py index b5f3fd7d8..b55d2c582 100644 --- a/apps/system_manage/models/__init__.py +++ b/apps/system_manage/models/__init__.py @@ -13,3 +13,4 @@ from .chat_user import * from .user_menu_setting import * from .access_control_policy import * from .ai_hub import * +from .platform_access import * diff --git a/apps/system_manage/models/platform_access.py b/apps/system_manage/models/platform_access.py new file mode 100644 index 000000000..f433896bb --- /dev/null +++ b/apps/system_manage/models/platform_access.py @@ -0,0 +1,62 @@ +"""Platform-wide roles and additive grants in the default workspace.""" +import uuid +from django.db import models +from django.contrib.postgres.fields import ArrayField + + +class PlatformRole(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=64, unique=True) + description = models.CharField(max_length=256, blank=True, default='') + enabled = models.BooleanField(default=True) + is_default = models.BooleanField(default=False) + modules = models.JSONField(default=list) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'platform_role' + + +class PlatformRoleMember(models.Model): + role = models.ForeignKey(PlatformRole, on_delete=models.CASCADE, related_name='members') + user = models.ForeignKey('users.User', on_delete=models.CASCADE, related_name='platform_roles') + + class Meta: + db_table = 'platform_role_member' + constraints = [models.UniqueConstraint(fields=['role', 'user'], name='unique_platform_role_member')] + + +class PlatformGrant(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + subject_type = models.CharField(max_length=8, choices=[('ALL', '所有用户'), ('ROLE', '角色'), ('USER', '用户')]) + subject_id = models.CharField(max_length=64, blank=True, default='') + workspace_id = models.CharField(max_length=128, default='default', db_index=True) + resource_type = models.CharField(max_length=32) + target = models.CharField(max_length=128) + actions = models.JSONField(default=list) + expires_at = models.DateTimeField(null=True, blank=True) + granted_by = models.ForeignKey('users.User', null=True, on_delete=models.SET_NULL, related_name='+') + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'platform_grant' + constraints = [models.UniqueConstraint(fields=['workspace_id', 'subject_type', 'subject_id', 'resource_type', 'target'], name='unique_platform_grant')] + indexes = [models.Index(fields=['subject_type', 'subject_id'], name='platform_grant_subject_idx')] + + +class EffectiveResourcePermission(models.Model): + """Read-only SQL view merging legacy, role, user and public resource grants.""" + id = models.UUIDField(primary_key=True) + workspace_id = models.CharField(max_length=128) + user_id = models.UUIDField() + auth_target_type = models.CharField(max_length=128) + target = models.CharField(max_length=128) + auth_type = models.CharField(max_length=64) + permission_list = ArrayField(models.CharField(max_length=256)) + create_time = models.DateTimeField() + update_time = models.DateTimeField() + + class Meta: + managed = False + db_table = 'effective_resource_permission' diff --git a/apps/system_manage/parsers/__init__.py b/apps/system_manage/parsers/__init__.py new file mode 100644 index 000000000..2c6039867 --- /dev/null +++ b/apps/system_manage/parsers/__init__.py @@ -0,0 +1,38 @@ +"""Stable application boundary for independently deployed parser engines.""" +from pathlib import Path + +MINERU_INPUTS = frozenset({'pdf', 'png', 'jpg', 'jpeg', 'webp', 'bmp', 'tiff', 'gif', 'jp2'}) +NATIVE_INPUTS = frozenset({'docx', 'doc', 'xls', 'xlsx', 'csv', 'html', 'htm', 'md', 'txt'}) + + +def extension(name): + return Path(name).suffix.lower().lstrip('.') + + +def parse(parser, name, content, options, headers, progress_callback=None): + from rest_framework.exceptions import ValidationError + kind = extension(name) + if kind in NATIVE_INPUTS: + from .native import parse as native_parse + if progress_callback: + progress_callback({'stage': '使用内置处理器解析文档', 'percent': None}) + return native_parse(name, content) + protocol = options.get('protocol', 'legacy') + if protocol == 'legacy': + if parser.endpoint.rstrip('/').endswith('/v1'): + raise ValidationError('原版接口地址不能以 /v1 结尾,请修改解析服务配置') + if kind not in MINERU_INPUTS: + raise ValidationError('当前解析服务不支持该文件格式') + from .legacy import parse as engine_parse + elif protocol == 'v1': + from system_manage.ai_hub_parser_v1 import parse as engine_parse + else: + raise ValidationError('不支持的解析接口类型') + result = engine_parse(parser, name, content, options, headers, progress_callback=progress_callback) + result.setdefault('source', {}) + result['source'] = {'engine': 'mineru', 'protocol': protocol, + 'configured_version': options.get('engine_version', ''), **result['source']} + return result + + +__all__ = ['parse', 'extension', 'MINERU_INPUTS', 'NATIVE_INPUTS'] diff --git a/apps/system_manage/parsers/catalog.py b/apps/system_manage/parsers/catalog.py new file mode 100644 index 000000000..fc4580e94 --- /dev/null +++ b/apps/system_manage/parsers/catalog.py @@ -0,0 +1,53 @@ +"""Shared parser capability catalogue; installed services remain explicit resources.""" +BACKENDS_2 = { + 'pipeline': '传统管道', 'hybrid-auto-engine': '混合本地引擎', 'vlm-auto-engine': 'VLM 本地引擎', + 'hybrid-http-client': '混合远程推理', 'vlm-http-client': 'VLM 远程推理', +} +BACKENDS_3 = { + 'pipeline': '传统管道', 'hybrid-engine': '混合本地引擎', 'vlm-engine': 'VLM 本地引擎', + 'hybrid-http-client': '混合远程推理', 'vlm-http-client': 'VLM 远程推理', +} +TIERS = {'flash': 'Flash 快速', 'basic': 'Basic 基础', 'standard': 'Standard 标准', 'advanced': 'Advanced 高精度'} + + +def catalogue(): + return {'versions': [{'id': '2.7.4', 'protocol': 'legacy', 'backends': BACKENDS_2}, + {'id': '3.4.5', 'protocol': 'legacy', 'backends': BACKENDS_3}, + {'id': '4.x', 'protocol': 'v1', 'backends': TIERS}], + 'builtin': {'id': 'builtin', 'name': 'MaxKB 内置解析', 'backend': 'builtin', + 'formats': ['PDF', 'DOCX', 'DOC', 'XLSX', 'XLS', 'CSV', 'HTML', 'MD', 'TXT']}} + + +def validate_options(options, endpoint): + from urllib.parse import urlsplit + from rest_framework.exceptions import ValidationError + if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier', 'engine_version', 'server_url', 'effort'}: + raise ValidationError('解析参数无效') + version = options.get('engine_version', '4.x' if options.get('protocol') == 'v1' else '2.7.4') + spec = next((v for v in catalogue()['versions'] if v['id'] == version), None) + if not spec or options.get('protocol', spec['protocol']) != spec['protocol']: + raise ValidationError('解析版本与接口不匹配') + if spec['protocol'] == 'legacy': + if urlsplit(endpoint).path.endswith('/v1'): + raise ValidationError('2.x / 3.x 请填写解析 API 地址,不含 /v1') + backend = options.get('backend', 'pipeline') + if not isinstance(backend, str): + raise ValidationError('解析后端无效') + aliases = {'vlm-auto-engine': 'vlm-engine', 'hybrid-auto-engine': 'hybrid-engine'} if version == '3.4.5' else {} + if aliases.get(backend, backend) not in spec['backends']: + raise ValidationError('该版本不支持此解析后端') + if backend.endswith('-http-client'): + if not isinstance(options.get('server_url'), str): + raise ValidationError('请填写有效的远程推理地址') + remote = urlsplit(options.get('server_url', '')) + if remote.scheme not in ('http', 'https') or not remote.hostname or remote.username or remote.password or remote.query or remote.fragment: + raise ValidationError('请填写有效的远程推理地址') + elif options.get('tier', 'standard') not in TIERS: + raise ValidationError('解析档位无效') + if options.get('effort', 'medium') not in ('medium', 'high'): + raise ValidationError('解析精度无效') + if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'): + raise ValidationError('识别语言无效') + if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options): + raise ValidationError('解析参数无效') + return {**options, 'engine_version': version, 'protocol': spec['protocol']} diff --git a/apps/system_manage/parsers/legacy.py b/apps/system_manage/parsers/legacy.py new file mode 100644 index 000000000..e03a2c8ca --- /dev/null +++ b/apps/system_manage/parsers/legacy.py @@ -0,0 +1,72 @@ +"""Official MinerU 2.x sync HTTP, executed inside the Hub's durable background task.""" +import json +import re +import time +import requests +from rest_framework.exceptions import ValidationError + + +def parse(parser, name, content, options, headers, progress_callback=None): + from system_manage.ai_hub import text, MAX_INLINE_TEXT + if progress_callback: + progress_callback({'stage': 'MinerU 正在解析文档', 'percent': None}) + backend = options.get('backend', 'pipeline') + if options.get('engine_version') == '3.4.5': + backend = {'hybrid-auto-engine': 'hybrid-engine', 'vlm-auto-engine': 'vlm-engine'}.get(backend, backend) + fields = {'backend': backend, 'lang_list': options.get('language', 'ch'), + 'parse_method': 'ocr' if options.get('ocr') else 'auto', 'return_md': 'true', + 'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true', + 'formula_enable': str(options.get('formula', True)).lower(), + 'table_enable': str(options.get('table', True)).lower()} + if backend.endswith('-http-client'): + fields['server_url'] = options['server_url'] + if options.get('engine_version') == '3.4.5': + fields['effort'] = options.get('effort', 'medium') + try: + response = requests.post(parser.endpoint.rstrip('/') + '/file_parse', files={'files': (name, content)}, headers=headers, + data=fields, timeout=(10, 300), allow_redirects=False) + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + if response.status_code == 202 and options.get('engine_version') == '3.4.5': + try: + task_id = response.json().get('task_id') + if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id): + raise ValidationError('解析服务返回任务 ID 无效') + deadline = time.monotonic() + 300 + while response.status_code == 202: + if time.monotonic() >= deadline: + raise ValidationError('解析等待超时,后台任务 ID:' + task_id) + if progress_callback: + progress_callback({'stage': '等待远程解析完成', 'percent': None}) + time.sleep(2) + response = requests.get(parser.endpoint.rstrip('/') + '/tasks/' + task_id + '/result', + headers=headers, timeout=(10, 30), allow_redirects=False) + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + except ValueError as exc: + raise ValidationError('解析任务响应无效') from exc + if response.status_code != 200: + raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + try: + payload = response.json() + except ValueError as exc: + raise ValidationError('解析服务返回格式无效') from exc + entries = payload.get('results') if isinstance(payload, dict) else None + if not isinstance(entries, dict) or len(entries) != 1: + raise ValidationError('解析结果格式无效') + result = next(iter(entries.values())) + if not isinstance(result, dict): + raise ValidationError('解析结果格式无效') + middle = result.get('middle_json') or {} + if isinstance(middle, str): + try: + middle = json.loads(middle) + except ValueError as exc: + raise ValidationError('解析结构数据无效') from exc + if not isinstance(middle, dict): + raise ValidationError('解析结构数据无效') + if progress_callback: + progress_callback({'stage': '整理解析结果和图片', 'percent': None}) + return {'markdown': text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT), + 'source': {'version': payload.get('version', ''), 'backend': payload.get('backend', ''), 'middle_json': middle}, + 'parser_id': str(parser.id), 'images': result.get('images', {})} diff --git a/apps/system_manage/parsers/native.py b/apps/system_manage/parsers/native.py new file mode 100644 index 000000000..7058d6b85 --- /dev/null +++ b/apps/system_manage/parsers/native.py @@ -0,0 +1,71 @@ +"""Reuse MaxKB's built-in converters; MinerU receives only supported visual documents.""" +import base64 +import io +import mimetypes +from pathlib import Path +import shutil +import subprocess +import tempfile +import zipfile +from rest_framework.exceptions import ValidationError + + +def parse(name, content): + from django.core.files.uploadedfile import SimpleUploadedFile + from system_manage.ai_hub import text + from . import extension + suffix = extension(name) + if suffix == 'doc': + command = shutil.which('libreoffice') or shutil.which('soffice') + if command is None: + raise ValidationError('DOC 格式需要 LibreOffice,请先另存为 DOCX') + with tempfile.TemporaryDirectory(prefix='maxkb-office-') as directory: + root = Path(directory); source = root / 'input.doc'; source.write_bytes(content) + try: + result = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(), + '--headless', '--convert-to', 'docx', '--outdir', directory, str(source)], + capture_output=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ValidationError('DOC 转换失败,请另存为 DOCX') from exc + output = root / 'input.docx' + if result.returncode or not output.is_file(): + raise ValidationError('DOC 转换失败,请另存为 DOCX') + parsed = parse(Path(name).stem + '.docx', output.read_bytes()) + parsed['source']['converted_from'] = 'doc' + return parsed + if suffix in ('docx', 'xlsx'): + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + if sum(item.file_size for item in archive.infolist()) > 200 * 1024 * 1024: + raise ValidationError('文档解压后超过大小限制') + except zipfile.BadZipFile as exc: + raise ValidationError('文件格式无效') from exc + converters = { + 'pdf': ('pdf_split_handle', 'PdfSplitHandle'), + 'docx': ('doc_split_handle', 'DocSplitHandle'), 'xlsx': ('xlsx_split_handle', 'XlsxSplitHandle'), + 'xls': ('xls_split_handle', 'XlsSplitHandle'), 'csv': ('csv_split_handle', 'CsvSplitHandle'), + 'html': ('html_split_handle', 'HTMLSplitHandle'), 'htm': ('html_split_handle', 'HTMLSplitHandle'), + 'md': ('text_split_handle', 'TextSplitHandle'), 'txt': ('text_split_handle', 'TextSplitHandle'), + } + if suffix not in converters: + raise ValidationError('内置处理器不支持该文件格式') + import importlib + module, converter = converters[suffix] + handler = getattr(importlib.import_module('common.handle.impl.text.' + module), converter)() + images = {} + def capture(files): + for file in files or []: + raw = file.meta.get('content') + if isinstance(raw, bytes): + mime = mimetypes.guess_type(file.file_name)[0] or 'image/jpeg' + images['./oss/file/' + str(file.id)] = f'data:{mime};base64,' + base64.b64encode(raw).decode() + upload = SimpleUploadedFile(Path(name.replace('\\', '/')).name, content) + result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0) + sections = result if isinstance(result, list) else [result] + markdown = '\n\n'.join('\n\n'.join( + ((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', '')) + for item in section.get('content', [])) for section in sections if isinstance(section, dict)) + for path, uri in images.items(): + markdown = markdown.replace(path, uri) + return {'markdown': text(markdown, '解析结果'), 'images': {}, + 'source': {'engine': 'maxkb-native', 'format': suffix}} diff --git a/apps/system_manage/platform_access.py b/apps/system_manage/platform_access.py new file mode 100644 index 000000000..a20c7c4f9 --- /dev/null +++ b/apps/system_manage/platform_access.py @@ -0,0 +1,350 @@ +"""Additive authorization: public + enabled roles + individual grants. + +The legacy permission table remains an independent source. New grants are never +materialized into it, so revocation cannot erase an owner's existing access. +""" +from django.core.exceptions import ValidationError as DjangoValidationError +import re +from django.db.models import Q +from django.db import transaction +from django.utils import timezone +from rest_framework.exceptions import PermissionDenied, ValidationError +from system_manage.models.platform_access import PlatformRole, PlatformRoleMember, PlatformGrant + +MODULES = {'application': 'AI应用', 'mindmap': '思维导图', 'knowledge': '知识库', 'tool': '工具管理', 'model': '模型管理'} +FEATURES = {'document_parse': '文档解析'} +ACTIONS = {'VIEW': '查看', 'USE': '使用', 'EDIT': '编辑', 'AUTHORIZE': '授权'} +DEFAULT_ROLE_ID = '00000000-0000-0000-0000-000000000001' +ADMIN_ROLE_ID = '00000000-0000-0000-0000-000000000002' +BUILTIN_ADMIN_USER_ID = 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab' + + +def is_admin(user): + return PlatformRoleMember.objects.filter(user_id=user.id, role_id=ADMIN_ROLE_ID, role__enabled=True).exists() + + +@transaction.atomic +def reconcile_builtin_roles(): + """One-time upgrade: import existing administrators without changing grants. + + Safe to repeat; only the initial creation imports legacy account types. + After that memberships are authoritative, including after demotion. + """ + from users.models import User + PlatformRole.objects.get_or_create(id=DEFAULT_ROLE_ID, defaults={ + 'name': '普通用户', 'description': '新激活用户自动关联', 'is_default': True, + 'modules': ['knowledge', 'mindmap']}) + admin_role, created = PlatformRole.objects.get_or_create(id=ADMIN_ROLE_ID, defaults={ + 'name': '系统管理员', 'description': '拥有全部平台管理权限', 'modules': list(MODULES)}) + if created: + PlatformRoleMember.objects.bulk_create([ + PlatformRoleMember(role=admin_role, user_id=pk) + for pk in User.objects.filter(role='ADMIN').values_list('id', flat=True) + ], ignore_conflicts=True) + return {'admin_role_created': created, + 'admin_members': PlatformRoleMember.objects.filter(role=admin_role).count()} + + +def _sync_legacy_role(user, operator_id=None): + """Keep old integrations as projections of the unified memberships.""" + from users.models import UserRoleAssignment + from users.serializers.user_manage_ext import invalidate_user_auth_cache + from users.serializers.user import update_user_role + role = 'ADMIN' if is_admin(user) else 'USER' + user.role = role + user.save(update_fields=['role', 'update_time']) + UserRoleAssignment.objects.filter(user_id=user.id, workspace_id='default', role_id__in=['ADMIN', 'USER']).delete() + UserRoleAssignment.objects.create(user_id=user.id, role_id=role, workspace_id='default', granted_by_id=operator_id) + if str(user.id) != BUILTIN_ADMIN_USER_ID: + update_user_role({'role_setting': [{'role_id': role, 'workspace_ids': ['None' if role == 'ADMIN' else 'default']}]}, user, operator_id) + # Clear immediately for callers in this transaction and again after commit + # to avoid another request repopulating caches with the previous state. + invalidate_user_auth_cache(user.id) + transaction.on_commit(lambda: invalidate_user_auth_cache(user.id)) + + +@transaction.atomic +def assign_roles(user_id, role_ids, operator_id=None): + from users.models import User + # All membership writers lock this row first, including bulk role editing. + PlatformRole.objects.select_for_update().get(pk=ADMIN_ROLE_ID) + user = User.objects.select_for_update().get(pk=user_id) + ids = {str(pk) for pk in role_ids} | {DEFAULT_ROLE_ID} + roles = list(PlatformRole.objects.select_for_update().filter(pk__in=ids).order_by('id')) + if len(roles) != len(ids): + raise ValidationError('角色不存在') + if str(user.id) == BUILTIN_ADMIN_USER_ID and ADMIN_ROLE_ID not in ids: + raise ValidationError('不能移除内置管理员角色') + if is_admin(user) and ADMIN_ROLE_ID not in ids and not PlatformRoleMember.objects.filter( + role_id=ADMIN_ROLE_ID, user__is_active=True).exclude(user_id=user.id).exists(): + raise ValidationError('至少保留一名启用的系统管理员') + PlatformRoleMember.objects.filter(user=user).exclude(role_id__in=ids).delete() + existing = {str(pk) for pk in user.platform_roles.values_list('role_id', flat=True)} + PlatformRoleMember.objects.bulk_create([PlatformRoleMember(user=user, role_id=pk) for pk in ids - existing]) + _sync_legacy_role(user, operator_id) + return user + + +def assign_legacy_role(user, role_id, operator_id=None): + """Adapter for registration and older clients; preserves custom roles.""" + if role_id not in ('ADMIN', 'USER'): + raise ValidationError('角色不合法') + ids = {str(pk) for pk in user.platform_roles.values_list('role_id', flat=True)} + ids.discard(ADMIN_ROLE_ID) + if role_id == 'ADMIN': + ids.add(ADMIN_ROLE_ID) + return assign_roles(user.id, ids, operator_id) + + +def require_admin(user): + if not is_admin(user): + raise PermissionDenied('仅管理员可管理角色和授权') + + +def roles_for(user): + return PlatformRole.objects.filter(enabled=True).filter(Q(is_default=True) | Q(members__user_id=user.id)).distinct() + + +def grants_for(user): + if not user.is_active: + return PlatformGrant.objects.none() + role_ids = [str(pk) for pk in roles_for(user).values_list('id', flat=True)] + return PlatformGrant.objects.filter(workspace_id='default').exclude(resource_type='FEATURE').filter( + Q(expires_at__isnull=True) | Q(expires_at__gt=timezone.now()) + ).filter(Q(subject_type='ALL') | Q(subject_type='USER', subject_id=str(user.id)) | + Q(subject_type='ROLE', subject_id__in=role_ids)) + + +def expand_actions(actions): + result = set(actions) + if 'EDIT' in result: + result |= {'VIEW', 'USE'} + if 'USE' in result or 'AUTHORIZE' in result: + result.add('VIEW') + return result + + +def resource_actions(user, resource_type, target, workspace='default'): + if not user.is_active or workspace != 'default': + return set() + if is_admin(user): + return set(ACTIONS) + actions = set() + for grant in grants_for(user).filter(resource_type=resource_type, target=str(target)): + actions.update(grant.actions) + return expand_actions(actions) + + +def module_list(user): + from system_manage.models import UserMenuSetting + if is_admin(user): + return list(MODULES) + result = set() + for role in roles_for(user): + result.update(role.modules) + # Preserve explicit legacy user menus; no implicit fallback grants once roles exist. + setting = UserMenuSetting.objects.filter(user_id=user.id).first() + if setting: + result.update(setting.menu_list or []) + for grant in grants_for(user).filter(resource_type='MODULE'): + if 'VIEW' in expand_actions(grant.actions): + result.add(grant.target) + return [key for key in MODULES if key in result] + + +def check_feature_request(request, user): + """Backend counterpart to menu visibility for business write operations. + + Read endpoints remain available for cross-module resource selectors; their + existing resource permission checks still apply. + """ + if is_admin(user) or request.method in ('GET', 'HEAD', 'OPTIONS'): + return + match = re.search(r'/workspace/default/(knowledge|application|tool|model)(?:/|$)', request.path) + if match and match.group(1) not in module_list(user): + from common.exception.app_exception import AppUnauthorizedFailed + raise AppUnauthorizedFailed(403, '未授权使用该功能') + + +def permission_strings(user): + """Expand new grants to the original endpoint permission vocabulary. + + Do not emit bare resource markers: legacy endpoints treat those as full + management. AUTHORIZE is separate from EDIT. + """ + from common.constants.permission_constants import (PermissionConstants, Resource, + ResourcePermission, ResourcePermissionGroup, get_permission_list_by_resource_group) + result = set() + for grant in grants_for(user).exclude(resource_type='MODULE'): + if grant.resource_type not in ('KNOWLEDGE', 'APPLICATION', 'TOOL', 'MODEL'): + continue + actions = expand_actions(grant.actions) + scope = f':/WORKSPACE/default/{grant.resource_type}/{grant.target}' + for level in ('VIEW', 'MANAGE'): + if level == 'VIEW' and 'VIEW' not in actions: + continue + if level == 'MANAGE' and 'EDIT' not in actions: + continue + group = ResourcePermissionGroup(Resource(grant.resource_type), ResourcePermission(level)) + for permission in get_permission_list_by_resource_group(group): + name = str(permission) + # Member/grant management requires AUTHORIZE, even for editors. + if any(part in name for part in ('WORKSPACE_USER_RESOURCE_PERMISSION', '+AUTH', '+ADD_MEMBER', '+REMOVE_MEMBER')): + continue + result.add(name + scope) + if 'AUTHORIZE' in actions: + for permission in PermissionConstants: + name = str(permission.value) + if name.startswith(grant.resource_type + '_WORKSPACE_USER_RESOURCE_PERMISSION:') or name.startswith(grant.resource_type + ':READ+AUTH'): + result.add(name + scope) + return result + + +def resource_models(): + from application.models import Application + from knowledge.models import Knowledge + from tools.models import Tool + from models_provider.models import Model + from system_manage.models.ai_hub import AIParserResource, AIService + return {'KNOWLEDGE': Knowledge, 'APPLICATION': Application, 'TOOL': Tool, + 'MODEL': Model, 'PARSER': AIParserResource, 'SERVICE': AIService} + + +def catalog(): + resources = [] + for kind, model in resource_models().items(): + fields = ['id', 'name'] + (['provider', 'model_type'] if kind == 'MODEL' else []) + for item in model.objects.filter(workspace_id='default').values(*fields).order_by('name'): + resources.append({**item, 'id': str(item['id']), 'resource_type': kind}) + return resources + + +def validate_target(resource_type, target): + if resource_type == 'MODULE': + if target not in MODULES: + raise ValidationError('功能无效') + elif resource_type == 'FEATURE': + if target not in FEATURES: + raise ValidationError('功能无效') + else: + model = resource_models().get(resource_type) + try: + valid = model is not None and model.objects.filter(pk=target, workspace_id='default').exists() + except (ValueError, TypeError, DjangoValidationError): + valid = False + if not valid: + raise ValidationError('资源不存在') + + +def effective_detail(user): + """Explain every source; legacy entries remain identifiable and revocable.""" + from system_manage.models import WorkspaceUserResourcePermission, UserMenuSetting + role_names = {str(r.id): r.name for r in roles_for(user)} + rows = [] + for grant in grants_for(user): + rows.append({'resource_type': grant.resource_type, 'target': grant.target, + 'actions': sorted(expand_actions(grant.actions)), + 'source': '所有用户' if grant.subject_type == 'ALL' else ( + role_names.get(grant.subject_id, '角色') if grant.subject_type == 'ROLE' else '单独授权'), + 'expires_at': grant.expires_at}) + for role in roles_for(user): + rows.extend({'resource_type': 'MODULE', 'target': module, 'actions': ['VIEW'], 'source': role.name} + for module in role.modules) + legacy_menu = UserMenuSetting.objects.filter(user_id=user.id).first() + if legacy_menu: + rows.extend({'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'source': '原用户菜单授权'} for m in legacy_menu.menu_list) + for grant in WorkspaceUserResourcePermission.objects.filter(user_id=user.id, workspace_id='default'): + actions = ['VIEW', 'USE'] if 'VIEW' in grant.permission_list else [] + if 'MANAGE' in grant.permission_list or 'ROLE' in grant.permission_list: + actions = list(ACTIONS) + rows.append({'resource_type': grant.auth_target_type, 'target': grant.target, 'actions': actions, 'source': '原用户资源授权'}) + for row in authorization_rows(): + if row.get('readonly') and ((row['subject_type'] == 'ROLE' and row['subject_id'] in role_names) or (row['subject_type'] == 'USER' and row['subject_id'] == str(user.id))): + rows.append({**row, 'source': role_names.get(row['subject_id'], '原用户菜单授权') + '(菜单附带)'}) + for kind in ('PARSER', 'SERVICE'): + model = resource_models()[kind] + for obj in model.objects.filter(workspace_id='default').filter(Q(owner_id=user.id) | Q(allowed_users__contains=[str(user.id)])): + owner = str(obj.owner_id) == str(user.id) + rows.append({'resource_type': kind, 'target': str(obj.id), 'actions': list(ACTIONS) if owner else ['VIEW', 'USE'], + 'source': '资源所有者' if owner else '原用户资源授权'}) + return {'modules': module_list(user), 'roles': [{'id': k, 'name': v} for k, v in role_names.items()], + 'permissions': rows, 'is_admin': is_admin(user)} + + +def authorization_rows(): + """Expose compatibility grants in the same management list, without migration.""" + from system_manage.models import UserMenuSetting, WorkspaceUserResourcePermission + rows = list(PlatformGrant.objects.values('id', 'subject_type', 'subject_id', 'resource_type', 'target', 'actions', 'expires_at')) + for role in PlatformRole.objects.exclude(id=ADMIN_ROLE_ID): + rows.extend({'id': f'role-menu:{role.id}:{m}', 'subject_type': 'ROLE', 'subject_id': str(role.id), + 'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'expires_at': None, + 'origin': '角色菜单'} for m in role.modules) + for setting in UserMenuSetting.objects.all(): + rows.extend({'id': f'user-menu:{setting.user_id}:{m}', 'subject_type': 'USER', 'subject_id': str(setting.user_id), + 'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'expires_at': None, + 'origin': '原菜单授权'} for m in setting.menu_list or []) + for g in WorkspaceUserResourcePermission.objects.filter(workspace_id='default'): + # Owner/framework ROLE rows are not discretionary grants. + if 'ROLE' in g.permission_list: + continue + actions = list(ACTIONS) if 'MANAGE' in g.permission_list else (['VIEW', 'USE'] if 'VIEW' in g.permission_list else []) + rows.append({'id': f'legacy-resource:{g.id}', 'subject_type': 'USER', 'subject_id': str(g.user_id), + 'resource_type': g.auth_target_type, 'target': g.target, 'actions': actions, + 'expires_at': None, 'origin': '原资源授权'}) + for kind in ('PARSER', 'SERVICE'): + for obj in resource_models()[kind].objects.filter(workspace_id='default'): + rows.extend({'id': f'legacy-share:{kind}:{obj.id}:{uid}', 'subject_type': 'USER', 'subject_id': uid, + 'resource_type': kind, 'target': str(obj.id), 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '原资源授权'} for uid in obj.allowed_users) + for role in PlatformRole.objects.exclude(id=ADMIN_ROLE_ID): + if {'tool', 'mindmap'} & set(role.modules): + rows.append({'id': f'inherited-feature:{role.id}', 'subject_type': 'ROLE', 'subject_id': str(role.id), + 'resource_type': 'FEATURE', 'target': 'document_parse', 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '菜单附带', 'readonly': True}) + for setting in UserMenuSetting.objects.all(): + if {'tool', 'mindmap'} & set(setting.menu_list or []): + rows.append({'id': f'inherited-user-feature:{setting.user_id}', 'subject_type': 'USER', 'subject_id': str(setting.user_id), + 'resource_type': 'FEATURE', 'target': 'document_parse', 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '菜单附带', 'readonly': True}) + return [row for row in rows if row['resource_type'] != 'FEATURE'] + + +def remove_compatibility_grant(key, expected=None): + from system_manage.models import UserMenuSetting, WorkspaceUserResourcePermission + row = next((r for r in authorization_rows() if str(r['id']) == key and r.get('origin')), None) + if row is None: + raise ValidationError('授权已变更,请刷新') + if row.get('readonly'): + raise ValidationError('此权限随原菜单生效,请调整菜单授权') + if expected and any(str(expected.get(k)) != str(row[k]) for k in ('subject_type', 'subject_id', 'resource_type', 'target')): + raise ValidationError('授权对象不匹配') + parts = key.split(':') + if parts[0] == 'role-menu': + role = PlatformRole.objects.select_for_update().get(pk=row['subject_id']) + role.modules = [m for m in role.modules if m != row['target']] + role.save(update_fields=['modules', 'updated_at']) + elif parts[0] == 'user-menu': + setting = UserMenuSetting.objects.select_for_update().get(user_id=row['subject_id']) + setting.menu_list = [m for m in setting.menu_list if m != row['target']] + setting.save(update_fields=['menu_list']) + elif parts[0] == 'legacy-resource': + WorkspaceUserResourcePermission.objects.filter(pk=parts[1]).delete() + from users.serializers.user_manage_ext import invalidate_user_auth_cache + transaction.on_commit(lambda: invalidate_user_auth_cache(row['subject_id'])) + elif parts[0] == 'legacy-share': + obj = resource_models()[row['resource_type']].objects.select_for_update().get(pk=row['target']) + obj.allowed_users = [uid for uid in obj.allowed_users if uid != row['subject_id']] + obj.save(update_fields=['allowed_users']) + + +def parser_feature_actions(user): + """Old tool/mindmap menus include parsing; new feature grants are explicit.""" + if not user.is_active: + return set() + if is_admin(user): + return {'VIEW', 'USE'} + from system_manage.models import UserMenuSetting + inherited = any({'tool', 'mindmap'} & set(role.modules) for role in roles_for(user)) + setting = UserMenuSetting.objects.filter(user_id=user.id).first() + inherited = inherited or bool(setting and {'tool', 'mindmap'} & set(setting.menu_list or [])) + return ({'VIEW', 'USE'} if inherited else set()) | resource_actions(user, 'FEATURE', 'document_parse') diff --git a/apps/system_manage/serializers/access_control_policy.py b/apps/system_manage/serializers/access_control_policy.py index e48f118a0..8c5b5b502 100644 --- a/apps/system_manage/serializers/access_control_policy.py +++ b/apps/system_manage/serializers/access_control_policy.py @@ -28,7 +28,7 @@ def _normalize_string_list(value): class AccessControlPolicySerializer(serializers.Serializer): - id = serializers.UUIDField(required=False) + id = serializers.UUIDField(read_only=True) name = serializers.CharField(required=True, max_length=128) description = serializers.CharField(required=False, allow_blank=True, default='') enabled = serializers.BooleanField(required=False, default=True) @@ -40,8 +40,8 @@ class AccessControlPolicySerializer(serializers.Serializer): allowed_regions = serializers.ListField(child=serializers.CharField(), required=False) effect_start_time = serializers.DateTimeField(required=False, allow_null=True) effect_end_time = serializers.DateTimeField(required=False, allow_null=True) - application_count = serializers.IntegerField(required=False) - last_applied_time = serializers.DateTimeField(required=False, allow_null=True) + application_count = serializers.IntegerField(read_only=True) + last_applied_time = serializers.DateTimeField(read_only=True) create_time = serializers.DateTimeField(required=False) update_time = serializers.DateTimeField(required=False) content = serializers.CharField(required=False) @@ -99,9 +99,14 @@ class AccessControlPolicySerializer(serializers.Serializer): return sorted(set(value or [])) def validate_allowed_device_types(self, value): + from system_manage.access_control import DEVICE_TYPES + if set(value) - set(DEVICE_TYPES): + raise AppApiException(500, '设备类型仅支持桌面端、移动终端、平板、服务器') return _normalize_string_list(value) def validate_allowed_regions(self, value): + if len(value) > 100 or any(not item.strip() or len(item.strip()) > 64 for item in value): + raise AppApiException(500, '区域名称须为1~64字,最多100项') return _normalize_string_list(value) def validate(self, attrs): @@ -113,8 +118,8 @@ class AccessControlPolicySerializer(serializers.Serializer): if access_time_enabled and (access_start_time is None or access_end_time is None): raise AppApiException(500, '启用访问时段限制后,必须设置开始和结束时间') - if access_start_time and access_end_time and access_start_time >= access_end_time: - raise AppApiException(500, '访问结束时间必须晚于开始时间') + if access_time_enabled and access_start_time == access_end_time: + raise AppApiException(500, '开始和结束时间不能相同') if effect_start_time and effect_end_time and effect_start_time >= effect_end_time: raise AppApiException(500, '生效结束时间必须晚于开始时间') return attrs @@ -151,28 +156,15 @@ class AccessControlPolicySerializer(serializers.Serializer): class AccessControlPolicyApplySerializer(serializers.Serializer): target_type = serializers.ChoiceField(choices=AccessControlPolicyApplicationRecord.TargetType.choices) - target_name = serializers.CharField(required=True, max_length=128) - target_identifier = serializers.CharField(required=False, allow_blank=True, default='', max_length=128) + target_name = serializers.CharField(required=False, allow_blank=True, default='', max_length=128) + target_identifier = serializers.CharField(required=True, max_length=128) remark = serializers.CharField(required=False, allow_blank=True, default='') @transaction.atomic def save(self, policy: AccessControlPolicy, operator): self.is_valid(raise_exception=True) - validated_data = self.validated_data - AccessControlPolicyApplicationRecord.objects.create( - policy=policy, - policy_name=policy.name, - target_type=validated_data.get('target_type'), - target_name=validated_data.get('target_name').strip(), - target_identifier=validated_data.get('target_identifier', '').strip(), - remark=validated_data.get('remark', '').strip(), - operator=operator, - operator_name=getattr(operator, 'nick_name', '') or getattr(operator, 'username', ''), - ) - policy.application_count = (policy.application_count or 0) + 1 - policy.last_applied_time = timezone.now() - policy.save(update_fields=['application_count', 'last_applied_time', 'update_time']) - return {'policy_id': str(policy.id), 'target_name': validated_data.get('target_name').strip()} + from system_manage.access_control import bind + return bind(policy.id, operator, self.validated_data) class AccessControlPolicyApplicationRecordSerializer(serializers.Serializer): diff --git a/apps/system_manage/serializers/menu_setting.py b/apps/system_manage/serializers/menu_setting.py index 2fac11776..7366a732a 100644 --- a/apps/system_manage/serializers/menu_setting.py +++ b/apps/system_manage/serializers/menu_setting.py @@ -133,6 +133,11 @@ class MenuSettingSerializer(serializers.Serializer): @staticmethod def get_current_menu_list(user): + from system_manage.platform_access import module_list + return module_list(user) + + @staticmethod + def get_legacy_menu_list(user): if MenuSettingSerializer._is_system_admin(user): return MenuSettingSerializer._get_admin_menu_list() user_id = getattr(user, 'id', user) diff --git a/apps/system_manage/serializers/system.py b/apps/system_manage/serializers/system.py index fe507e34e..c42c0330a 100644 --- a/apps/system_manage/serializers/system.py +++ b/apps/system_manage/serializers/system.py @@ -48,9 +48,5 @@ class SystemProfileSerializer(serializers.Serializer): class SystemConfigSerializer(serializers.Serializer): @staticmethod def get_config(): - # Prefer the bundled workbench: it uses the current login through its parent. - bundled = os.path.join(settings.STATIC_ROOT, 'admin', 'mindmap-workbench', 'index.html') - return { - 'mindmap_url': (CONFIG.get_admin_path() + '/mindmap-workbench/index.html') - if os.path.isfile(bundled) else CONFIG.get('MINDMAP_URL', '') - } + # Kept for older clients; the workbench is now a native MaxKB route. + return {'mindmap_url': CONFIG.get_admin_path() + '/mindmap'} diff --git a/apps/system_manage/serializers/user_resource_permission.py b/apps/system_manage/serializers/user_resource_permission.py index 9ce375ec3..e51251a9b 100644 --- a/apps/system_manage/serializers/user_resource_permission.py +++ b/apps/system_manage/serializers/user_resource_permission.py @@ -157,6 +157,9 @@ class UserResourcePermissionSerializer(serializers.Serializer): workspace_manage = is_workspace_manage(user_id, workspace_id) if workspace_manage: return True + from system_manage.platform_access import resource_actions + if 'USE' in resource_actions(User.objects.get(pk=user_id), auth_target_type, resource_id, workspace_id): + return True wurp = QuerySet(WorkspaceUserResourcePermission).filter(auth_target_type=auth_target_type, workspace_id=workspace_id, user=user_id, target=resource_id).first() diff --git a/apps/system_manage/test_access_control.py b/apps/system_manage/test_access_control.py new file mode 100644 index 000000000..53cd3a2c4 --- /dev/null +++ b/apps/system_manage/test_access_control.py @@ -0,0 +1,234 @@ +from datetime import datetime, time, timedelta, timezone as tz +from unittest import TestCase +from unittest.mock import patch +from types import SimpleNamespace +from django.db import transaction +from django.test import RequestFactory +from rest_framework.exceptions import ValidationError +from common.exception.app_exception import AppUnauthorizedFailed, AppApiException +from system_manage import access_control as ac +from system_manage.models import AccessControlPolicy, AccessControlPolicyApplicationRecord, Log +from system_manage.serializers.access_control_policy import AccessControlPolicySerializer +from users.models import User + + +class AccessTests(TestCase): + def setUp(self): + self.atomic = transaction.atomic() + self.atomic.__enter__() + self.user = User.objects.create(username='CODEX_ACCESS_TEST', nick_name='测试管理员', role='ADMIN') + self.policy = AccessControlPolicy.objects.create(name='CODEX_ACCESS_TEST') + self.now = datetime(2026, 10, 26, 2, 0, tzinfo=tz.utc) + + def tearDown(self): + transaction.set_rollback(True) + self.atomic.__exit__(None, None, None) + + def request(self, ip='10.1.2.3', agent='Mozilla/5.0 (Windows NT 10.0)', **meta): + return RequestFactory().get('/chat/api/open', REMOTE_ADDR=ip, HTTP_USER_AGENT=agent, **meta) + + def bind(self, target='application:sample'): + with patch.object(ac, 'targets', return_value=[{'id': target, 'label': '测试服务'}]): + return ac.bind(self.policy.id, self.user, {'target_type': 'SERVICE', 'target_identifier': target}) + + def test_no_binding_does_not_restrict_or_audit(self): + ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.count(), 0) + + def test_existing_history_does_not_activate(self): + AccessControlPolicyApplicationRecord.objects.create(policy=self.policy, policy_name=self.policy.name, + target_type='SERVICE', target_identifier='application:sample', target_name='历史记录') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.count(), 0) + + def test_service_binding_and_deduplicated_audit(self): + self.bind(); request = self.request() + ac.enforce(request, 'application', 'sample'); ac.enforce(request, 'application', 'sample') + self.assertEqual(Log.objects.count(), 1) + self.assertEqual(Log.objects.first().status, 200) + + def test_device_denied(self): + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.first().status, 403) + self.assertNotIn('body', Log.objects.first().details) + + def test_update_applies_to_next_request(self): + self.bind(); ac.enforce(self.request(), 'application', 'sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_disabled_policy(self): + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.enabled = False; self.policy.save() + ac.enforce(self.request(), 'application', 'sample') + + def test_unbind_keeps_history(self): + item = self.bind(); ac.unbind(self.policy.id, self.user, item) + self.assertEqual(ac.configuration()[1]['bindings'], []) + self.assertEqual(AccessControlPolicyApplicationRecord.objects.count(), 2) + + def test_duplicate_binding(self): + self.bind() + with self.assertRaises(ValidationError): self.bind() + + def test_nonexistent_service(self): + with patch.object(ac, 'targets', return_value=[]), self.assertRaises(ValidationError): + ac.bind(self.policy.id, self.user, {'target_type': 'SERVICE', 'target_identifier': 'application:missing'}) + + def test_device_cidr(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.2.3/24'}) + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'mindmap', 'sample') + ac.enforce(self.request(ip='10.2.2.3'), 'mindmap', 'sample') + + def test_ipv6(self): + self.assertTrue(ac.contains('2001:db8::1', '2001:db8::/32')) + self.assertFalse(ac.contains('10.1.1.1', '2001:db8::/32')) + + def test_invalid_network(self): + for value in ['invalid', None, '10.500.0.1']: + with self.subTest(value=value), self.assertRaises(ValidationError): ac.network(value) + + def test_all_policies_must_allow(self): + self.bind() + other = AccessControlPolicy.objects.create(name='另一策略', allowed_device_types=['移动终端']) + row, config = ac.configuration() + config['bindings'].append({**config['bindings'][0], 'policy_id': str(other.id)}) + row.meta = config; row.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_unknown_region(self): + self.bind(); self.policy.allowed_regions = ['北京']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_region_mapping_and_change(self): + self.bind(); self.policy.allowed_regions = ['北京']; self.policy.save() + ac.save_regions([{'name': '北京', 'network': '10.1.0.0/16'}]) + ac.enforce(self.request(), 'application', 'sample') + ac.save_regions([]) + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_overlapping_regions(self): + with self.assertRaises(ValidationError): + ac.save_regions([{'name': '北京', 'network': '10.0.0.0/8'}, {'name': '上海', 'network': '10.1.0.0/16'}]) + + def test_spoofed_forwarded_ignored(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': ''}): + self.assertEqual(ac.client_ip(self.request(HTTP_X_FORWARDED_FOR='8.8.8.8')), '10.1.2.3') + + def test_trusted_proxy_chain(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32,10.0.0.0/8'}): + self.assertEqual(ac.client_ip(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='1.1.1.1, 8.8.8.8, 10.2.2.2')), '8.8.8.8') + + def test_invalid_forwarded_fails_closed(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32'}): + with self.assertRaises(AppUnauthorizedFailed): + ac.client_ip(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='invalid')) + + def test_invalid_proxy_address_cannot_bypass_device_policy(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.0.0/16'}) + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32'}): + with self.assertRaises(AppUnauthorizedFailed): + ac.enforce(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='invalid'), 'mindmap', 'sample') + self.assertEqual(Log.objects.first().details['reason'], '来源 IP 无效') + + def test_device_classification(self): + for agent, expected in [('Mozilla Windows', '桌面端'), ('iPhone Mobile', '移动终端'), + ('iPad', '平板'), ('Android Tablet', '平板'), ('curl/8', '服务器'), ('', '未知')]: + with self.subTest(agent=agent): self.assertEqual(ac.device_type(self.request(agent=agent)), expected) + + def test_time_boundaries(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(10); self.policy.access_end_time = time(11) + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + self.assertTrue(ac.policy_reason(self.policy, self.now + timedelta(hours=1), '桌面端', set())) + + def test_weekday(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(9); self.policy.access_end_time = time(18) + self.policy.allowed_weekdays = [2] + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + + def test_overnight_start_day(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(22); self.policy.access_end_time = time(6) + self.policy.allowed_weekdays = [1] + for utc_hour in [15, 19]: + self.assertEqual(ac.policy_reason(self.policy, self.now.replace(hour=utc_hour), '桌面端', set()), '') + self.assertTrue(ac.policy_reason(self.policy, self.now.replace(hour=22), '桌面端', set())) + + def test_effective_boundaries(self): + self.policy.allowed_device_types = ['移动终端'] + self.policy.effect_start_time = self.now + timedelta(seconds=1) + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + self.policy.effect_start_time = self.now + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + self.policy.effect_end_time = self.now + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + + def test_invalid_time_denied(self): + self.policy.access_time_enabled = True + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + + def test_serializer_overnight_and_equal(self): + values = {'name': '夜间策略', 'access_time_enabled': True, 'access_start_time': '22:00', 'access_end_time': '06:00'} + self.assertTrue(AccessControlPolicySerializer(data=values).is_valid(raise_exception=True)) + values['access_end_time'] = '22:00' + with self.assertRaises(AppApiException): AccessControlPolicySerializer(data=values).is_valid(raise_exception=True) + + def test_serializer_device_options(self): + with self.assertRaises(AppApiException): + AccessControlPolicySerializer(data={'name': '无效设备策略', 'allowed_device_types': ['任意设备']}).is_valid(raise_exception=True) + + def test_auth_does_not_read_lazy_user(self): + self.bind() + class PendingRequest: + META = {'REMOTE_ADDR': '10.1.2.3', 'HTTP_USER_AGENT': 'Mozilla'} + path = '/chat/api/chat' + @property + def user(self): raise AssertionError('authentication recursion') + ac.enforce(PendingRequest(), 'application', 'sample') + + def test_token_auth_enforces_published_and_api_requests(self): + from common.auth.authenticate import TokenAuth + from common.constants.permission_constants import ChatAuth + auth = ChatAuth([], [], 'client', 'APPLICATION_API_KEY', 'sample') + handler = SimpleNamespace(support=lambda *args: True, handle=lambda *args: (None, auth)) + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.save() + request = self.request(); request.META['HTTP_AUTHORIZATION'] = 'Bearer test' + with patch('common.auth.authenticate.handles', [handler]), patch('traceback.print_stack'), self.assertRaises(AppUnauthorizedFailed): + TokenAuth().authenticate(request) + + def test_token_cannot_call_another_application(self): + from common.auth.authenticate import TokenAuth + from common.constants.permission_constants import ChatAuth + auth = ChatAuth([], [], 'client', 'APPLICATION_API_KEY', 'sample') + handler = SimpleNamespace(support=lambda *args: True, handle=lambda *args: (None, auth)) + request = self.request(); request.META['HTTP_AUTHORIZATION'] = 'Bearer test' + request.resolver_match = SimpleNamespace(kwargs={'application_id': 'different'}) + with patch('common.auth.authenticate.handles', [handler]), patch('traceback.print_stack'), self.assertRaises(AppUnauthorizedFailed): + TokenAuth().authenticate(request) + + def test_direct_workbench_cannot_bypass_device_binding(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.0.0/16'}) + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce_task(self.request(), nodes=[{'type': 'render'}]) + + def test_retry_checks_original_service(self): + self.bind(target='mindmap:sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce_task(self.request(), 'sample', [{'type': 'render'}]) + + def test_parser_check_precedes_file_storage(self): + from unittest.mock import Mock + from django.core.files.uploadedfile import SimpleUploadedFile + from knowledge.serializers.document import DocumentSerializers + self.bind(target='parser:sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + serializer = DocumentSerializers.Split(data={}, context={'request': self.request()}) + serializer._validated_data = {} + serializer._errors = {} + serializer._data = {'workspace_id': 'default', 'knowledge_id': 'sample'} + serializer.is_valid = Mock(return_value=True) + with patch('knowledge.serializers.document.QuerySet') as query, patch('knowledge.models.File.save') as store: + query.return_value.filter.return_value.first.return_value = SimpleNamespace(meta={'parser_id': 'sample'}) + with self.assertRaises(AppUnauthorizedFailed): serializer.parse({'file': [SimpleUploadedFile('CODEX.pdf', b'test')]}) + store.assert_not_called() diff --git a/apps/system_manage/test_access_control_decisions.py b/apps/system_manage/test_access_control_decisions.py new file mode 100644 index 000000000..28cc56a4c --- /dev/null +++ b/apps/system_manage/test_access_control_decisions.py @@ -0,0 +1,72 @@ +from datetime import datetime, timedelta, timezone +from unittest import TestCase + +from django.db import transaction +from rest_framework.exceptions import ValidationError + +from system_manage.access_control_decisions import list_decisions +from system_manage.models import Log + + +class DecisionTests(TestCase): + def setUp(self): + self.atomic = transaction.atomic() + self.atomic.__enter__() + self.start = datetime(2026, 10, 27, 0, 0, tzinfo=timezone.utc) + for index in range(235): + denied = index % 2 == 0 + row = Log.objects.create(menu='接入控制', operate='拒绝访问' if denied else '允许访问', + status=403 if denied else 200, ip_address='192.0.2.1' if denied else '2001:db8::1', + operation_object={'target': 'CODEX_DECISION_TEST'}, + details={'device_type': '移动终端' if denied else '桌面端', + 'reason': '设备类型不允许' if denied else '', 'id': 'untrusted', 'status': 999}) + Log.objects.filter(pk=row.id).update(create_time=self.start + timedelta(minutes=index)) + Log.objects.create(menu='其他菜单', operate='拒绝访问', status=403, ip_address='192.0.2.1') + Log.objects.create(menu='接入控制', operate='更新区域网段', status=200, ip_address='192.0.2.1') + + def tearDown(self): + transaction.set_rollback(True) + self.atomic.__exit__(None, None, None) + + def test_pages_cover_beyond_old_limit(self): + pages = [list_decisions({'page': page, 'page_size': 100}) for page in (1, 2, 3)] + self.assertEqual([len(page['records']) for page in pages], [100, 100, 35]) + self.assertEqual({page['total'] for page in pages}, {235}) + self.assertEqual(len({row['id'] for page in pages for row in page['records']}), 235) + self.assertTrue(pages[0]['records'][0]['time'] > pages[1]['records'][0]['time']) + self.assertEqual(pages[0]['records'][0]['status'], 403) + + def test_combined_filters_and_timezone_boundaries(self): + page = list_decisions({'start_time': '2026-10-27T08:10:00', + 'end_time': '2026-10-27T08:20:00+08:00', 'result': 'denied', + 'ip': '192.0.2', 'device_type': '移动终端', 'reason': '设备'}) + self.assertEqual(page['total'], 6) + self.assertEqual(page['records'][0]['time'], self.start + timedelta(minutes=20)) + self.assertEqual(page['records'][-1]['time'], self.start + timedelta(minutes=10)) + + def test_filters_independently(self): + for params, total in [({'result': 'allowed'}, 117), ({'denied': 'true'}, 118), + ({'ip': 'db8::'}, 117), ({'device_type': '移动终端'}, 118), + ({'reason': '不允许'}, 118), ({'device_type': '未知'}, 0)]: + with self.subTest(params=params): + self.assertEqual(list_decisions(params)['total'], total) + + def test_empty_and_out_of_range_pages(self): + empty = list_decisions({'ip': 'absent', 'page': 99}) + self.assertEqual((empty['records'], empty['total'], empty['current']), ([], 0, 1)) + self.assertEqual(list_decisions({'page': 99, 'page_size': 100})['current'], 3) + + def test_reject_invalid_parameters(self): + for params in [{'page': 0}, {'page_size': 101}, {'page': 'abc'}, {'result': 'other'}, + {'device_type': 'other'}, {'start_time': 'invalid'}, + {'start_time': '2026-10-28T00:00:00', 'end_time': '2026-10-27T00:00:00'}]: + with self.subTest(params=params), self.assertRaises(ValidationError): + list_decisions(params) + + def test_stable_order_for_equal_times(self): + Log.objects.filter(menu='接入控制').update(create_time=self.start) + first = list_decisions({'page_size': 100}) + second = list_decisions({'page': 2, 'page_size': 100}) + ids = [row['id'] for row in first['records'] + second['records']] + self.assertEqual(ids, sorted(ids, reverse=True)) + self.assertEqual(len(set(ids)), 200) diff --git a/apps/system_manage/test_ai_hub.py b/apps/system_manage/test_ai_hub.py index 494ac8ed6..3d8be5770 100644 --- a/apps/system_manage/test_ai_hub.py +++ b/apps/system_manage/test_ai_hub.py @@ -102,7 +102,7 @@ class HubTests(TestCase): import requests from types import SimpleNamespace from system_manage.views.ai_hub import ParserTest - p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1'}) + p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}}) with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id) get.assert_not_called() get.return_value = Mock(status_code=302) @@ -149,7 +149,25 @@ class HubTests(TestCase): task.refresh_from_db() self.assertEqual(task.status, 'SUCCESS') self.assertEqual(task.result['original'], '# 内容\n数值42') - self.assertEqual(task.result['source']['pages'], [1]) + self.assertEqual(task.result['source']['middle_json']['pages'], [1]) + + def test_builtin_docx_works_without_external_parser(self): + import io + from docx import Document as WordDocument + document = WordDocument() + document.add_heading('Native42', 0) + document.add_paragraph('保留原文数值42') + buffer = io.BytesIO(); document.save(buffer) + task = hub.submit('default', self.user, {'nodes': [ + {'type': 'parse', 'processor': 'builtin'}, {'type': 'render'}]}, + SimpleUploadedFile('native.docx', buffer.getvalue())) + with patch('requests.post') as external: + execute(claim()) + external.assert_not_called() + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS', task.error) + self.assertIn('42', task.result['markdown']) + self.assertEqual(task.result['source']['engine'], 'maxkb-native') def test_parser_disabled_or_changed_after_submit_fails(self): parser = self.parser() @@ -171,6 +189,19 @@ class HubTests(TestCase): self.assertEqual(task.result['original'], '# 原稿') self.assertEqual(task.result['markdown'], '# 整理稿') + def test_model_params_form_defaults_win_over_absent_node_values(self): + from models_provider.models import Model + model = Model.objects.create(name='默认参数', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', + model_params_form=[{'field': 'temperature', 'default_value': 0.7}, {'field': 'max_tokens', 'default_value': 8192}]) + task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]}) + fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿') + with patch('models_provider.tools.get_model', return_value=fake) as get: + execute(claim()) + self.assertEqual(get.call_args.kwargs['temperature'], 0.7) + self.assertEqual(get.call_args.kwargs['max_tokens'], 8192) + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS') + def test_provider_exception_is_sanitized(self): parser = self.parser() task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf')) diff --git a/apps/system_manage/test_ai_hub_organize.py b/apps/system_manage/test_ai_hub_organize.py new file mode 100644 index 000000000..3da33f69c --- /dev/null +++ b/apps/system_manage/test_ai_hub_organize.py @@ -0,0 +1,44 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch +from system_manage.ai_hub_organize import organize, split_markdown, token_bound + + +class OrganizeTests(unittest.TestCase): + def test_split_preserves_long_unbroken_unicode_and_image_tokens(self): + text = '# 标题\n' + '超长段落42' * 8000 + '\nAIHUBIMAGE0001TOKEN\n结束' + chunks = split_markdown(text, 2048) + self.assertEqual(''.join(chunks), text) + self.assertTrue(all(token_bound(chunk) <= 2048 for chunk in chunks)) + self.assertTrue(any('AIHUBIMAGE0001TOKEN' in chunk for chunk in chunks)) + + def test_long_document_is_organized_and_merged_with_checkpoints(self): + source = '\n'.join(f'## section {i}\n' + 'fact42 ' * 1000 for i in range(12)) + stages, saved, calls = [], [], [] + def invoke(prompt, content): + calls.append(content) + return SimpleNamespace(content='# Result\n- fact42', response_metadata={}) + result = organize(source, '整理事实', invoke, context_tokens=8192, output_tokens=1024, + progress=stages.append, checkpoint=saved.append) + self.assertIn('fact42', result) + self.assertGreater(len(calls), 2) + self.assertGreater(len(saved), 2) + self.assertTrue(any('合并' in stage['stage'] for stage in stages)) + + def test_truncated_output_is_not_accepted_as_success(self): + with self.assertRaisesRegex(ValueError, '截断'): + organize('facts', '整理', lambda *args: SimpleNamespace( + content='partial', response_metadata={'finish_reason': 'length'})) + + def test_failed_chunk_keeps_completed_checkpoints_and_sanitizes_provider_error(self): + saved, calls = [], [] + def invoke(*args): + calls.append(1) + if len(calls) > 1: + raise RuntimeError('private credential must not escape') + return '# saved42' + with patch('system_manage.ai_hub_organize.time.sleep'): + with self.assertRaisesRegex(ValueError, '已保留') as failure: + organize('word42 ' * 5000, '整理', invoke, context_tokens=4096, output_tokens=512, checkpoint=saved.append) + self.assertNotIn('private', str(failure.exception)) + self.assertEqual(saved, [['# saved42']]) diff --git a/apps/system_manage/test_document_export.py b/apps/system_manage/test_document_export.py new file mode 100644 index 000000000..a69f88c2b --- /dev/null +++ b/apps/system_manage/test_document_export.py @@ -0,0 +1,89 @@ +"""No database or external model is needed for these export contract checks.""" +import base64 +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +from docx import Document +from PIL import Image +from system_manage.document_export import export_document, ExportError + + +class DocumentExportTests(unittest.TestCase): + def test_real_docx_preserves_structure_and_merged_cells(self): + image = io.BytesIO() + Image.new('RGB', (40, 20), 'blue').save(image, format='PNG') + html = '

中文报告

加粗斜体2

' + html += '
  1. 第三项
' + html += '

可编辑链接测试图

' + html += '
表头
合并A
B

结束标记

' + raw, mime = export_document(html) + self.assertTrue(raw.startswith(b'PK')) + self.assertIn('wordprocessingml', mime) + doc = Document(io.BytesIO(raw)) + self.assertEqual(len(doc.inline_shapes), 1) + self.assertEqual(len(doc.tables[0].rows), 3) + self.assertEqual(doc.tables[0].cell(1, 0).text, doc.tables[0].cell(2, 0).text) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + xml = archive.read('word/document.xml').decode() + for expected in ('中文报告', '结束标记', 'w:numPr', 'w:hyperlink', 'w:tblHeader', 'w:vMerge', 'w:gridSpan', 'w:vertAlign'): + self.assertIn(expected, xml) + + def test_image_resolution_never_fetches_external_urls(self): + for src in ('https://example.com/private', 'file:///etc/passwd', 'data:image/png;base64,AAAA'): + with self.assertRaises(ExportError): + export_document('

') + + def test_unsupported_format_and_paper_are_rejected(self): + for kwargs in ({'format_name': 'html'}, {'paper': 'unknown'}, {'paper': 'Letter'}): + with self.assertRaises(ExportError): export_document('

正文

', **kwargs) + + def test_long_table_grid_is_not_rebuilt_per_cell(self): + from docx.table import Table + read_grid = Table._cells.fget + calls = [] + def counted(table): + calls.append(1) + return read_grid(table) + html = '' + ''.join( + f'' for i in range(400)) + '
编号内容
{i}完整内容 {i}
' + with patch.object(Table, '_cells', property(counted)): + raw, _ = export_document(html) + self.assertEqual(len(calls), 1) + doc = Document(io.BytesIO(raw)) + self.assertEqual(len(doc.tables[0].rows), 401) + self.assertEqual(doc.tables[0].cell(400, 1).text, '完整内容 399') + section = doc.sections[0] + self.assertAlmostEqual(section.page_width.mm, 210, places=1) + self.assertAlmostEqual(section.page_height.mm, 297, places=1) + + def test_bad_table_cannot_overflow_column_limit(self): + with self.assertRaises(ExportError): + export_document('' + '' * 31 + '
x
') + + def test_conversion_failure_is_explicit(self): + with patch('system_manage.document_export.shutil.which', return_value=None): + with self.assertRaisesRegex(ExportError, '转换组件不可用'): + export_document('

正文

', 'pdf') + + def test_conversion_uses_private_profile_and_removes_temporary_files(self): + directories = [] + def convert(args, **kwargs): + root = Path(args[args.index('--outdir') + 1]); directories.append(root) + self.assertTrue(any(arg.startswith('-env:UserInstallation=file:') for arg in args)) + self.assertEqual(kwargs['timeout'], 90) + self.assertNotIn('shell', kwargs) + (root / 'document.pdf').write_bytes(b'%PDF-1.7\ncontract-test') + return type('Result', (), {'returncode': 0})() + with patch('system_manage.document_export.shutil.which', return_value='soffice'), patch('system_manage.document_export.subprocess.run', side_effect=convert): + raw, mime = export_document('

正文

', 'pdf') + self.assertEqual(mime, 'application/pdf') + self.assertTrue(raw.startswith(b'%PDF')) + self.assertTrue(all(not path.exists() for path in directories)) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/system_manage/urls.py b/apps/system_manage/urls.py index f3f2cef21..72cbea963 100644 --- a/apps/system_manage/urls.py +++ b/apps/system_manage/urls.py @@ -1,11 +1,20 @@ from django.urls import path from . import views -from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport +from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport, DocumentExport +from .views.platform_access import AccessView, RoleView, GrantsView, UserAccessView +from .views.access_control_policy import AccessControlRuntimeView, AccessControlDecisionView app_name = "system_manage" # @formatter:off urlpatterns = [ + path('platform_access', AccessView.as_view()), + path('platform_access/roles', RoleView.as_view()), + path('platform_access/roles/', RoleView.as_view()), + path('platform_access/grants', GrantsView.as_view()), + path('platform_access/grants/', GrantsView.as_view()), + path('platform_access/users/', UserAccessView.as_view()), + path('workspace//ai_hub/document_export', DocumentExport.as_view()), path('workspace//ai_hub/exports', MarkdownExport.as_view()), path('workspace//ai_hub/assets//content', AssetContent.as_view()), path('workspace//ai_hub/parsers//test', ParserTest.as_view()), @@ -20,6 +29,8 @@ urlpatterns = [ path('menu_setting', views.MenuSettingView.as_view()), path('menu_setting/current', views.CurrentMenuSettingView.as_view()), path('access_control_policy', views.AccessControlPolicyView.as_view()), + path('access_control_policy/runtime', AccessControlRuntimeView.as_view()), + path('access_control_policy/decisions', AccessControlDecisionView.as_view()), path('access_control_policy/application_record', views.AccessControlPolicyApplicationRecordView.as_view()), path('access_control_policy/', views.AccessControlPolicyDetailView.as_view()), path('access_control_policy//apply', views.AccessControlPolicyApplyView.as_view()), diff --git a/apps/system_manage/views/access_control_policy.py b/apps/system_manage/views/access_control_policy.py index 0d5eaaf35..b69c20c53 100644 --- a/apps/system_manage/views/access_control_policy.py +++ b/apps/system_manage/views/access_control_policy.py @@ -9,6 +9,7 @@ from drf_spectacular.utils import extend_schema from rest_framework.request import Request from rest_framework.views import APIView +from django.db import transaction from common.auth import TokenAuth from common.auth.authentication import has_permissions @@ -34,7 +35,7 @@ class AccessControlPolicyView(APIView): authentication_classes = [TokenAuth] @extend_schema(tags=['Access Control Policy']) - @has_permissions(PermissionConstants.USER_READ, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def get(self, request: Request): return result.success( AccessControlPolicySerializer.list( @@ -45,7 +46,7 @@ class AccessControlPolicyView(APIView): @extend_schema(tags=['Access Control Policy']) @log(menu='Policy management', operate='Create access control policy', get_details=get_access_control_policy_details) - @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def post(self, request: Request): return result.success(AccessControlPolicySerializer(data=request.data).save()) @@ -54,7 +55,7 @@ class AccessControlPolicyDetailView(APIView): authentication_classes = [TokenAuth] @extend_schema(tags=['Access Control Policy']) - @has_permissions(PermissionConstants.USER_READ, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def get(self, request: Request, policy_id): return result.success( AccessControlPolicySerializer.to_representation_from_instance( @@ -64,17 +65,23 @@ class AccessControlPolicyDetailView(APIView): @extend_schema(tags=['Access Control Policy']) @log(menu='Policy management', operate='Update access control policy', get_details=get_access_control_policy_details) - @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def put(self, request: Request, policy_id): instance = AccessControlPolicySerializer.get_one(policy_id) return result.success(AccessControlPolicySerializer(instance=instance, data=request.data).save()) @extend_schema(tags=['Access Control Policy']) @log(menu='Policy management', operate='Delete access control policy', get_details=get_access_control_policy_details) - @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def delete(self, request: Request, policy_id): - instance = AccessControlPolicySerializer.get_one(policy_id) - instance.delete() + from system_manage.access_control import configuration + with transaction.atomic(): + instance = AccessControlPolicy.objects.select_for_update().get(pk=policy_id) + row, config = configuration(lock=True) + config['bindings'] = [b for b in config.get('bindings', []) if b['policy_id'] != str(instance.id)] + row.meta = config + row.save(update_fields=['meta', 'update_time']) + instance.delete() return result.success(True) @@ -83,7 +90,7 @@ class AccessControlPolicyApplyView(APIView): @extend_schema(tags=['Access Control Policy']) @log(menu='Policy management', operate='Apply access control policy', get_details=get_access_control_policy_details) - @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def post(self, request: Request, policy_id): policy = AccessControlPolicySerializer.get_one(policy_id) serializer = AccessControlPolicyApplySerializer(data=request.data) @@ -94,7 +101,7 @@ class AccessControlPolicyApplicationRecordView(APIView): authentication_classes = [TokenAuth] @extend_schema(tags=['Access Control Policy']) - @has_permissions(PermissionConstants.USER_READ, RoleConstants.ADMIN) + @has_permissions(RoleConstants.ADMIN) def get(self, request: Request): return result.success( AccessControlPolicyApplicationRecordSerializer.list( @@ -102,3 +109,39 @@ class AccessControlPolicyApplicationRecordView(APIView): target_name=request.query_params.get('target_name', ''), ) ) + + +from system_manage.models import AccessControlPolicy +from system_manage import access_control as access + + +class AccessControlRuntimeView(APIView): + authentication_classes = [TokenAuth] + + @has_permissions(RoleConstants.ADMIN) + def get(self, request): + _, config = access.configuration() + names = {str(p.id): p.name for p in AccessControlPolicy.objects.all()} + return result.success({**config, 'bindings': [{**b, 'policy_name': names.get(b['policy_id'], '')} + for b in config.get('bindings', [])], 'targets': access.targets(), + 'device_types': access.DEVICE_TYPES, 'timezone': 'Asia/Shanghai'}) + + @has_permissions(RoleConstants.ADMIN) + @log(menu='接入控制', operate='更新区域网段', get_details=get_access_control_policy_details) + def put(self, request): + return result.success(access.save_regions(request.data.get('region_networks'))) + + @has_permissions(RoleConstants.ADMIN) + @log(menu='接入控制', operate='解除策略绑定', get_details=get_access_control_policy_details) + def delete(self, request): + access.unbind(request.data.get('policy_id'), request.user, request.data) + return result.success(True) + + +class AccessControlDecisionView(APIView): + authentication_classes = [TokenAuth] + + @has_permissions(RoleConstants.ADMIN) + def get(self, request): + from system_manage.access_control_decisions import list_decisions + return result.success(list_decisions(request.query_params)) diff --git a/apps/system_manage/views/ai_hub.py b/apps/system_manage/views/ai_hub.py index 43ae45d82..9889b16bb 100644 --- a/apps/system_manage/views/ai_hub.py +++ b/apps/system_manage/views/ai_hub.py @@ -14,26 +14,47 @@ from system_manage.models.ai_hub import AIService, AITask class HubView(APIView): authentication_classes = [TokenAuth] + def delete(self, request, workspace_id, kind, pk=None): + if not pk: + raise ValidationError('请选择要删除的记录') + hub.delete_record(kind, pk, workspace_id, request.user) + return result.success(True) + def initial(self, request, *args, **kwargs): super().initial(request, *args, **kwargs) from users.serializers.user import get_workspace_list_by_user if not request.user.is_active or kwargs['workspace_id'] not in {str(w['id']) for w in get_workspace_list_by_user(request.user.id)}: raise PermissionDenied('无权访问') + from system_manage.platform_access import module_list + modules = module_list(request.user) + parser_request = kwargs.get('kind') in ('parsers', 'resources', 'parser_catalog') or isinstance(self, ParserTest) + if parser_request: + if not {'tool', 'mindmap'} & set(modules): + raise PermissionDenied('未授权使用文档解析') + elif 'mindmap' not in modules: + raise PermissionDenied('未授权使用思维导图') def get(self, request, workspace_id, kind, pk=None): + if kind == 'parser_catalog': + from system_manage.parsers.catalog import catalogue + return result.success(catalogue()) if kind == 'resources': parsers = hub.visible(hub.AIParserResource.objects.filter(workspace_id=workspace_id, enabled=True), request.user) from users.models import User from users.models.user_manage_ext import UserActivationApplication pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True) - return result.success({'parsers': [hub.serialize(p) for p in parsers], + from system_manage.platform_access import grants_for + may_authorize = hub.admin(request.user) or grants_for(request.user).filter( + resource_type__in=['PARSER', 'SERVICE'], actions__contains=['AUTHORIZE']).exists() + return result.success({'parsers': [hub.serialize(p, user=request.user) for p in parsers + if 'USE' in hub.serialize(p, user=request.user)['actions']], 'models': list(hub.authorized_models(workspace_id, request.user).values('id', 'name', 'model_type')), - 'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if hub.admin(request.user) else [], + 'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if may_authorize else [], 'is_admin': hub.admin(request.user)}) if pk: obj = hub.one(kind, pk, workspace_id, request.user) - data = hub.serialize(obj, detail=True) - if kind == 'parsers' and hub.admin(request.user): + data = hub.serialize(obj, detail=True, user=request.user) + if kind == 'parsers' and 'EDIT' in data.get('actions', []): data['endpoint'] = obj.endpoint return result.success(data) if kind not in hub.KINDS: @@ -43,7 +64,7 @@ class HubView(APIView): query = query.defer('file_data', 'input_data', 'snapshot', 'result') elif kind == 'artifacts': query = query.defer('original', 'markdown', 'history') - return result.success([hub.serialize(obj) for obj in query.order_by('-created_at')[:200]]) + return result.success([hub.serialize(obj, user=request.user) for obj in query.order_by('-created_at')[:200]]) def post(self, request, workspace_id, kind, pk=None): data = request.data @@ -55,15 +76,33 @@ class HubView(APIView): raise ValidationError('任务参数无效') if not isinstance(data, dict): raise ValidationError('请求参数无效') + if kind == 'tasks': + from system_manage.access_control import enforce_task + if pk: + candidate = hub.one('tasks', pk, workspace_id, request.user, edit=True) + enforce_task(request, candidate.service_id, candidate.snapshot.get('nodes')) + else: + nodes = hub.one('flows', data['flow_id'], workspace_id, request.user).nodes if data.get('flow_id') else data.get('nodes') + enforce_task(request, nodes=nodes) if pk: with transaction.atomic(): - obj = hub.one(kind, pk, workspace_id, request.user, edit=True) + if kind == 'tasks': + # Match submit/delete lock order so a retry cannot race a + # service deletion and become a pending, detached task. + task = hub.one(kind, pk, workspace_id, request.user, edit=True) + if task.service_id: + hub.one('services', task.service_id, workspace_id, request.user, lock=True) + authorization_only = kind in ('parsers', 'services') and set(data) <= {'allowed_users'} + obj = hub.one(kind, pk, workspace_id, request.user, edit=not authorization_only, lock=True) if kind == 'parsers': obj = hub.parser_save(workspace_id, request.user, data, obj) elif kind == 'artifacts': obj = hub.artifact_save(workspace_id, request.user, data, obj) elif kind == 'services': - hub.require_admin(request.user) + if set(data) - {'allowed_users'}: + hub.require_resource_action(request.user, obj, 'EDIT') + if 'allowed_users' in data: + hub.require_resource_action(request.user, obj, 'AUTHORIZE') obj = AIService.objects.select_for_update().get(pk=obj.pk) if 'name' in data: obj.name = hub.text(data['name'], '名称', limit=128) @@ -110,7 +149,7 @@ class HubView(APIView): # Bound pending backlog per user; GPU work is consumed by the isolated worker. if AITask.objects.filter(owner_id=request.user.id, status__in=['PENDING', 'RUNNING']).count() >= 10: raise ValidationError('待处理任务已达上限') - obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file')) + obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), request=request) elif kind == 'artifacts': obj = hub.artifact_save(workspace_id, request.user, data) else: @@ -122,8 +161,10 @@ class ParserTest(HubView): http_method_names = ['post', 'options'] def post(self, request, workspace_id, pk): - hub.require_admin(request.user) parser = hub.one('parsers', pk, workspace_id, request.user) + hub.require_resource_action(request.user, parser, 'EDIT') + from system_manage.access_control import enforce + enforce(request, 'parser', parser.id) return result.success(hub.parser_test(parser)) @@ -149,6 +190,13 @@ class MarkdownExport(HubView): from django.http import HttpResponse from system_manage.ai_hub_assets import ASSET, asset_lock, image_bytes, normalize markdown = hub.text(request.data.get('markdown'), '文档内容') + import re + filename = request.data.get('filename', 'document.md') + if not isinstance(filename, str): raise ValidationError('文件名无效') + filename = filename.replace('\\', '/').rsplit('/', 1)[-1] + filename = re.sub(r'[<>:\"/\\|?*\x00-\x1f]', '_', filename).strip(' .')[:180] + filename = (filename[:-3] if filename.lower().endswith('.md') else filename).strip(' .') or 'document' + filename += '.md' with asset_lock(): markdown = normalize(markdown, workspace_id, request.user) buffer = io.BytesIO() @@ -168,7 +216,7 @@ class MarkdownExport(HubView): replacements[match[0]] = name for url, name in replacements.items(): markdown = markdown.replace(url, name) - archive.writestr('document.md', markdown) + archive.writestr(filename, markdown) response = HttpResponse(buffer.getvalue(), content_type='application/zip') response['Content-Disposition'] = 'attachment; filename="markdown.zip"' return response @@ -185,6 +233,9 @@ class PublishedRun(HubView): supplied = hashlib.sha256(request.headers.get('X-Service-Key', '').encode()).hexdigest() if not secrets.compare_digest(supplied, service.key_hash): raise PermissionDenied('服务Key无效') + from system_manage.access_control import enforce_task + hub.require_use(request.user, service) + enforce_task(request, service.id, service.flow.nodes) minute = int(timezone.now().timestamp()) // 60 key = f'ai-hub-rate:{service.id}:{minute}' cache.add(key, 0, 70) @@ -197,5 +248,34 @@ class PublishedRun(HubView): data = json.loads(request.data.get('data', '{}')) except (ValueError, TypeError): raise ValidationError('任务参数无效') - task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service) + task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service, request=request) return result.success(hub.serialize(task)) + + +class DocumentExport(HubView): + http_method_names = ['post', 'options'] + + def post(self, request, workspace_id): + from django.http import HttpResponse + from system_manage.document_export import export_document, ExportError, MAX_HTML + from system_manage.ai_hub_assets import ASSET, image_bytes + def image(src): + match = ASSET.fullmatch(src) + if not match or match[1] != workspace_id: + raise ExportError('请使用当前工作空间已保存的图片') + raw, _meta = image_bytes(workspace_id, request.user, match[2]) + return raw + format_name = request.data.get('format', 'docx') + try: + upload = request.FILES.get('content') + if upload and upload.size > MAX_HTML: raise ExportError('导出内容超过24MB') + html = upload.read().decode('utf-8') if upload else request.data.get('html', '') + raw, mime = export_document(html, format_name, + image_resolver=image, paper=request.data.get('paper', 'A4')) + except (ExportError, ValueError, TypeError) as exc: + raise ValidationError(str(exc) if isinstance(exc, ExportError) else '文档内容格式无效') from exc + response = HttpResponse(raw, content_type=mime) + response['Content-Disposition'] = 'attachment; filename="document.' + format_name + '"' + response['Cache-Control'] = 'private, no-store' + response['X-Content-Type-Options'] = 'nosniff' + return response diff --git a/apps/system_manage/views/platform_access.py b/apps/system_manage/views/platform_access.py new file mode 100644 index 000000000..0ab85c3db --- /dev/null +++ b/apps/system_manage/views/platform_access.py @@ -0,0 +1,197 @@ +from django.db import transaction +from django.shortcuts import get_object_or_404 +from django.utils import timezone +from rest_framework import serializers +from rest_framework.views import APIView +from rest_framework.exceptions import ValidationError +from common.auth import TokenAuth +from common.result import result +from common.log.log import log +from system_manage.models.platform_access import PlatformRole, PlatformRoleMember, PlatformGrant +from system_manage import platform_access as access +from users.models import User + + +class RoleInput(serializers.ModelSerializer): + modules = serializers.ListField(child=serializers.ChoiceField(choices=list(access.MODULES)), required=False) + user_ids = serializers.ListField(child=serializers.UUIDField(), required=False, write_only=True) + + class Meta: + model = PlatformRole + fields = ['name', 'description', 'enabled', 'modules', 'user_ids'] + + +class GrantInput(serializers.ModelSerializer): + legacy_id = serializers.CharField(required=False, write_only=True) + actions = serializers.ListField(child=serializers.ChoiceField(choices=list(access.ACTIONS)), allow_empty=False) + + class Meta: + model = PlatformGrant + fields = ['subject_type', 'subject_id', 'resource_type', 'target', 'actions', 'expires_at', 'legacy_id'] + validators = [] + + def validate(self, data): + if data['resource_type'] == 'FEATURE': + raise ValidationError('暂不支持功能授权') + subject = data['subject_type'] + key = data.get('subject_id', '') + if subject == 'ALL': + if key: + raise ValidationError('公共授权无需指定用户或角色') + else: + try: + key = str(serializers.UUIDField().run_validation(key)) + except serializers.ValidationError: + raise ValidationError('授权对象无效') + if subject == 'ROLE': + exists = PlatformRole.objects.filter(pk=key).exists() + else: + from users.serializers.user import exclude_unactivated_users + exists = exclude_unactivated_users(User.objects.filter(pk=key)).exists() + if not exists: + raise ValidationError('授权对象不存在或未激活') + data['subject_id'] = key + access.validate_target(data['resource_type'], data['target']) + if data['resource_type'] == 'FEATURE' and set(data['actions']) - {'VIEW', 'USE'}: + raise ValidationError('功能仅支持查看、使用权限') + if data['resource_type'] == 'MODULE' and set(data['actions']) != {'VIEW'}: + raise ValidationError('功能入口仅支持查看权限') + if data.get('expires_at') and data['expires_at'] <= timezone.now(): + raise ValidationError('有效期须晚于当前时间') + data['actions'] = sorted(set(data['actions'])) + return data + + +def role_data(role): + return {'id': str(role.id), 'name': role.name, 'description': role.description, + 'enabled': role.enabled, 'is_default': role.is_default, 'modules': role.modules, + 'is_builtin': str(role.id) in (access.DEFAULT_ROLE_ID, access.ADMIN_ROLE_ID), + 'is_admin_role': str(role.id) == access.ADMIN_ROLE_ID, + 'user_ids': [str(pk) for pk in role.members.values_list('user_id', flat=True)]} + + +def valid_users(ids): + ids = set(ids) + # Membership can be prepared for a disabled account; authentication still blocks it. + if User.objects.filter(pk__in=ids).count() != len(ids): + raise ValidationError('用户不存在') + return ids + + +class AccessView(APIView): + authentication_classes = [TokenAuth] + + def initial(self, request, *args, **kwargs): + super().initial(request, *args, **kwargs) + access.require_admin(request.user) + + def get(self, request): + from users.serializers.user import exclude_unactivated_users + users = list(exclude_unactivated_users(User.objects.all()).values('id', 'username', 'nick_name', 'role')) + return result.success({'roles': [role_data(r) for r in PlatformRole.objects.order_by('created_at')], + 'users': users, 'resources': access.catalog(), 'modules': access.MODULES, 'actions': access.ACTIONS, + 'grants': access.authorization_rows()}) + + +class RoleView(AccessView): + http_method_names = ['post', 'delete', 'options'] + @log(menu='Role management', operate='Save role') + @transaction.atomic + def post(self, request, pk=None): + PlatformRole.objects.select_for_update().get(pk=access.ADMIN_ROLE_ID) + role = get_object_or_404(PlatformRole.objects.select_for_update(), pk=pk) if pk else None + serializer = RoleInput(instance=role, data=request.data) + serializer.is_valid(raise_exception=True) + data = dict(serializer.validated_data) + members = data.pop('user_ids', None) + if role and role.is_default and data.get('enabled', True) is False: + raise ValidationError('默认角色不可停用') + if role and str(role.id) == access.ADMIN_ROLE_ID: + if any(data.get(key, getattr(role, key)) != getattr(role, key) for key in ('name', 'enabled', 'modules')): + raise ValidationError('系统管理员的名称、权限和启用状态不可修改') + if members is not None: + members = valid_users(members) + if role is None: + role = PlatformRole.objects.create(**data) + else: + for key, value in data.items(): + setattr(role, key, value) + role.save() + if members is not None: + existing = set(role.members.values_list('user_id', flat=True)) + for user_id in sorted(existing | members, key=str): + ids = {str(pk) for pk in PlatformRoleMember.objects.filter(user_id=user_id).values_list('role_id', flat=True)} + if user_id in members: + ids.add(str(role.id)) + else: + ids.discard(str(role.id)) + access.assign_roles(user_id, ids, request.user.id) + return result.success(role_data(role)) + + @log(menu='Role management', operate='Delete role') + @transaction.atomic + def delete(self, request, pk): + role = get_object_or_404(PlatformRole.objects.select_for_update(), pk=pk) + if role.is_default or str(role.id) == access.ADMIN_ROLE_ID: + raise ValidationError('内置角色不可删除') + PlatformGrant.objects.filter(subject_type='ROLE', subject_id=str(role.id)).delete() + role.delete() + return result.success(True) + + +class GrantsView(AccessView): + http_method_names = ['post', 'delete', 'options'] + @log(menu='Resource authorization', operate='Save grants') + @transaction.atomic + def post(self, request): + # Validate the entire batch before changing anything. + data = request.data.get('grants') + if not isinstance(data, list) or not 1 <= len(data) <= 500: + raise ValidationError('请选择授权资源(最多500项)') + serializer = GrantInput(data=data, many=True) + serializer.is_valid(raise_exception=True) + for item in serializer.validated_data: + values = dict(item) + legacy_id = values.pop('legacy_id', None) + if legacy_id: + access.remove_compatibility_grant(legacy_id, values) + elif values['resource_type'] == 'MODULE' and values['subject_type'] in ('ROLE', 'USER'): + key = ('role-menu:' if values['subject_type'] == 'ROLE' else 'user-menu:') + values['subject_id'] + ':' + values['target'] + if any(str(r['id']) == key for r in access.authorization_rows()): + access.remove_compatibility_grant(key, values) + subject_type = values['subject_type'] + # Serialize with role deletion so no dangling role grant is created. + if subject_type == 'ROLE': + get_object_or_404(PlatformRole.objects.select_for_update(), pk=values['subject_id']) + defaults = {k: values.pop(k, None) for k in ('actions', 'expires_at')} + PlatformGrant.objects.update_or_create(workspace_id='default', **values, + defaults={**defaults, 'granted_by': request.user}) + return result.success(True) + + @log(menu='Resource authorization', operate='Revoke grant') + @transaction.atomic + def delete(self, request, pk): + if ':' in str(pk): + access.remove_compatibility_grant(str(pk)) + return result.success(True) + pk = serializers.UUIDField().run_validation(pk) + get_object_or_404(PlatformGrant, pk=pk).delete() + return result.success(True) + + +class UserAccessView(AccessView): + http_method_names = ['get', 'put', 'options'] + def get(self, request, user_id): + user = get_object_or_404(User, pk=user_id) + data = access.effective_detail(user) + data['user_id'] = str(user.id) + data['role_ids'] = list({str(pk) for pk in PlatformRoleMember.objects.filter(user=user).values_list('role_id', flat=True)} | {access.DEFAULT_ROLE_ID}) + return result.success(data) + + @log(menu='Role management', operate='Assign user roles') + @transaction.atomic + def put(self, request, user_id): + ids = serializers.ListField(child=serializers.UUIDField(), max_length=100).run_validation(request.data.get('role_ids')) + get_object_or_404(User, pk=user_id) + access.assign_roles(user_id, ids, request.user.id) + return self.get(request, user_id) diff --git a/apps/tools/sql/list_tool_user.sql b/apps/tools/sql/list_tool_user.sql index 7b4a00ffa..7158d08a4 100644 --- a/apps/tools/sql/list_tool_user.sql +++ b/apps/tools/sql/list_tool_user.sql @@ -22,7 +22,7 @@ FROM (SELECT tool."id"::text, FROM (SELECT tool.* FROM tool tool ${tool_query_set} AND tool.id::text IN (SELECT target - FROM workspace_user_resource_permission + FROM effective_resource_permission ${workspace_user_resource_permission_query_set} AND 'VIEW' = ANY (permission_list))) AS tool LEFT JOIN "user" ON "user".id = user_id diff --git a/apps/tools/sql/list_tool_user_ee.sql b/apps/tools/sql/list_tool_user_ee.sql index 63af6cee0..aa624fe9f 100644 --- a/apps/tools/sql/list_tool_user_ee.sql +++ b/apps/tools/sql/list_tool_user_ee.sql @@ -22,7 +22,7 @@ FROM (SELECT tool."id"::text, FROM (SELECT tool.* FROM tool tool ${tool_query_set} AND tool.id::text IN (SELECT target - FROM workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + FROM effective_resource_permission ${workspace_user_resource_permission_query_set} AND CASE WHEN auth_type = 'ROLE' THEN 'ROLE' = ANY (permission_list) @@ -30,8 +30,8 @@ FROM (SELECT tool."id"::text, 'TOOL:READ' IN (SELECT (CASE WHEN user_role_relation.role_id = ANY (ARRAY ['USER']) THEN 'TOOL:READ' ELSE role_permission.permission_id END) FROM role_permission role_permission RIGHT JOIN user_role_relation user_role_relation ON user_role_relation.role_id=role_permission.role_id - WHERE user_role_relation.user_id=workspace_user_resource_permission.user_id - AND user_role_relation.workspace_id=workspace_user_resource_permission.workspace_id) + WHERE user_role_relation.user_id=effective_resource_permission.user_id + AND user_role_relation.workspace_id=effective_resource_permission.workspace_id) ELSE 'VIEW' = ANY (permission_list) END diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index e01ad1f68..fe3ee7cb6 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -77,6 +77,10 @@ class CreateUserSerializer(serializers.Serializer): def is_workspace_manage(user_id: str, workspace_id: str): + from system_manage.platform_access import is_admin + user = User.objects.filter(pk=user_id).first() + if user and is_admin(user): + return True workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping") role_permission_mapping_model = DatabaseModelManage.get_model("role_permission_mapping_model") is_x_pack_ee = workspace_user_role_mapping_model is not None and role_permission_mapping_model is not None @@ -84,7 +88,7 @@ def is_workspace_manage(user_id: str, workspace_id: str): return QuerySet(workspace_user_role_mapping_model).select_related('role', 'user').filter( workspace_id=workspace_id, user_id=user_id, role__type=RoleConstants.WORKSPACE_MANAGE.value.__str__()).exists() - return QuerySet(User).filter(id=user_id, role=RoleConstants.ADMIN.value.__str__()).exists() + return False def get_workspace_list_by_user(user_id): @@ -105,16 +109,8 @@ class UserProfileSerializer(serializers.Serializer): @return: """ workspace_list = get_workspace_list_by_user(user.id) - user_role_relation_model = DatabaseModelManage.get_model("workspace_user_role_mapping") - role_name = [user.role] - if user_role_relation_model: - user_role_relations = ( - user_role_relation_model.objects - .filter(user_id=user.id) - .select_related('role') - .distinct('role_id') - ) - role_name = [relation.role.role_name for relation in user_role_relations] + from system_manage.platform_access import roles_for + role_name = list(roles_for(user).values_list('name', flat=True)) return { 'id': user.id, @@ -132,6 +128,12 @@ class UserProfileSerializer(serializers.Serializer): class UserInstanceSerializer(serializers.ModelSerializer): + def to_representation(self, instance): + from system_manage.platform_access import roles_for + data = super().to_representation(instance) + data['platform_roles'] = [{'id': str(role.id), 'name': role.name} for role in roles_for(instance)] + return data + class Meta: model = User fields = ['id', 'username', 'email', 'phone', 'is_active', 'role', 'nick_name', 'create_time', 'update_time', diff --git a/apps/users/serializers/user_manage_ext.py b/apps/users/serializers/user_manage_ext.py index 1aa1b6e06..989575746 100644 --- a/apps/users/serializers/user_manage_ext.py +++ b/apps/users/serializers/user_manage_ext.py @@ -14,7 +14,6 @@ from common.exception.app_exception import AppApiException from common.utils.logger import maxkb_logger from system_manage.models import SystemSetting, SettingType from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment -from users.serializers.user import update_user_role ROLE_OPTIONS = [ {"id": RoleConstants.USER.name, "name": "普通用户"}, @@ -119,6 +118,9 @@ class ActivationSerializer(serializers.Serializer): @transaction.atomic def approve(application_id: str, audit_user_id: str, role_id: str = RoleConstants.USER.name, workspace_id: str = "default"): + from system_manage.models.platform_access import PlatformRole + from system_manage.platform_access import ADMIN_ROLE_ID + PlatformRole.objects.select_for_update().get(pk=ADMIN_ROLE_ID) application = (UserActivationApplication.objects.select_for_update() .select_related("user").filter(id=application_id).first()) if application is None: @@ -135,15 +137,11 @@ class ActivationSerializer(serializers.Serializer): user = application.user user.is_active = True - user.role = role_id user.save() - - # 审核通过时同步写入角色授权关系,形成独立的授权数据对象 - UserRoleAssignment.objects.filter(user_id=user.id).delete() - UserRoleAssignment.objects.create( - user_id=user.id, role_id=role_id, - workspace_id=workspace_id or "default", granted_by_id=audit_user_id - ) + from system_manage.platform_access import assign_legacy_role, require_admin + if role_id == RoleConstants.ADMIN.name: + require_admin(User.objects.get(pk=audit_user_id)) + assign_legacy_role(user, role_id, audit_user_id) invalidate_user_auth_cache(user.id) transaction.on_commit(lambda: send_activation_notice(user, True)) return {"id": str(application.id), "status": application.status, "user_id": str(user.id)} @@ -178,47 +176,31 @@ class RoleAssignmentSerializer(serializers.Serializer): user = User.objects.filter(id=user_id).first() if user is None: raise AppApiException(500, "用户不存在") - assignments = UserRoleAssignment.objects.filter(user_id=user_id).order_by("granted_at") - records = [{ - "role_id": item.role_id, - "workspace_id": item.workspace_id, - "granted_at": _fmt(item.granted_at), - } for item in assignments] - if not records: - records = [{"role_id": user.role, "workspace_id": "default", "granted_at": ""}] + from system_manage.platform_access import roles_for + from system_manage.models.platform_access import PlatformRole + records = [{'role_id': str(role.id), 'name': role.name, 'workspace_id': 'default', 'granted_at': ''} for role in roles_for(user)] return { "user": {"id": str(user.id), "username": user.username, "nick_name": user.nick_name, "email": user.email or "", "role": user.role, "is_active": user.is_active}, "assignments": records, - "available_roles": ROLE_OPTIONS, + "available_roles": [{'id': str(r.id), 'name': r.name} for r in PlatformRole.objects.order_by('created_at')], } @staticmethod @transaction.atomic def save(user_id: str, role_settings, granted_by_id: str): - user = User.objects.select_for_update().filter(id=user_id).first() + user = User.objects.filter(id=user_id).first() if user is None: raise AppApiException(500, "用户不存在") - if not isinstance(role_settings, list) or len(role_settings) != 1 or not isinstance(role_settings[0], dict): - raise AppApiException(500, "请选择一个角色") - item = role_settings[0] - role_id = item.get("role_id") - if role_id not in VALID_ROLE_IDS: - raise AppApiException(500, "角色只能为普通用户或管理员") - if item.get('workspace_ids') not in (None, [], ['default']): + if not isinstance(role_settings, list) or not all(isinstance(item, dict) for item in role_settings): raise AppApiException(500, '角色配置无效') - if str(user.id) == 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab': - if role_id != RoleConstants.ADMIN.name: - raise AppApiException(500, '不能修改内置管理员角色') - return RoleAssignmentSerializer.detail(str(user.id)) - UserRoleAssignment.objects.filter(user_id=user.id).delete() - UserRoleAssignment.objects.create( - user_id=user.id, role_id=role_id, workspace_id='default', granted_by_id=granted_by_id - ) - user.role = role_id - user.save() - update_user_role({'role_setting': [{'role_id': role_id, 'workspace_ids': [ - 'None' if role_id == RoleConstants.ADMIN.name else 'default' - ]}]}, user, granted_by_id) - invalidate_user_auth_cache(user.id) + from system_manage.platform_access import assign_roles, assign_legacy_role, require_admin + require_admin(User.objects.get(pk=granted_by_id)) + if any(item.get('workspace_ids') not in (None, [], ['default']) for item in role_settings): + raise AppApiException(500, '角色配置无效') + if len(role_settings) == 1 and role_settings[0].get('role_id') in VALID_ROLE_IDS: + assign_legacy_role(user, role_settings[0]['role_id'], granted_by_id) + else: + ids = serializers.ListField(child=serializers.UUIDField(), max_length=100).run_validation([item.get('role_id') for item in role_settings]) + assign_roles(user.id, ids, granted_by_id) return RoleAssignmentSerializer.detail(str(user.id)) diff --git a/scripts/README_native_mindmap.md b/scripts/README_native_mindmap.md new file mode 100644 index 000000000..ebc69622c --- /dev/null +++ b/scripts/README_native_mindmap.md @@ -0,0 +1,32 @@ +# 原生思维导图与文档解析 + +思维导图工作台位于 `ui/src/views/mindmap/workbench`,通过 MaxKB API 使用当前用户和工作空间,不再嵌入外部页面。旧 `MINDMAP_URL` 不再作为页面入口。 + +## 责任边界 + +- MaxKB:资源授权、同步/异步解析器适配、格式分流、任务与成果、智能整理、图片附件和预览。 +- UnisMindMap(本地仓库名 markitdown):官方 `mineru-2.7.4-released` 解析源码、运行依赖及厂商镜像。`mineru/` 保持官方发布包内容。 +- 解析器适配位于 `apps/system_manage/parsers`;现有 4.x v1 适配保持独立。调用方只消费统一 Markdown、图片和来源信息。 + +## 格式分流 + +- PDF、受支持图片:调用登记的 MinerU 服务。 +- DOCX、XLS/XLSX、CSV、HTML、Markdown、TXT:复用 MaxKB 内置处理器。工作台的内置解析无需外部解析资源;知识库继续原有分段和附件流程。 +- DOC:有 LibreOffice 时在独立临时目录转换为 DOCX;不可用时提示另存 DOCX。 +- 未引入微软 MarkItDown,也不宣称当前支持 PPT、OFD、EPUB 等格式。 + +PDF/图片的解析图片会转为知识库附件,沿用知识库读取权限及保留机制;导图图片继续使用 AI Hub 私有资产和离线导出。 + +## 解析资源 + +官方 2.7.4 使用根地址(如 `http://127.0.0.1:8100`),`protocol=legacy`,`engine_version=2.7.4`;调用 `/file_parse`。旧项目 `/api` 前缀不属于官方 API。 + +4.x 使用登记的 `/v1` 地址和 `protocol=v1`。不能仅改下拉框而保留不相符的地址,也不将四档参数映射为旧版引擎。 + +业务任务继续由现有单消费者执行。2.7.4 上游同步接口不提供细粒度任务进度,页面显示真实处理阶段和耗时,不模拟百分比。长文整理按上下文分块、合并;失败时保留原稿和已完成分块。 + +## 部署 + +无新增数据库迁移。切换前检查活动任务、备份资源配置及源文件;保留原解析器环境作为回退。不要覆盖厂商 Torch/vLLM。NVIDIA 与天数的验证结果分别记录。 + +前端统一构建:`npm run build-only -- --base=/admin/ --outDir=dist-native-mindmap`。发布至 8080 实际使用的 `apps/static/admin`,先上传资源、最后替换入口;不再单独构建或发布 markitdown/web_ui。 diff --git a/scripts/README_release_review.md b/scripts/README_release_review.md new file mode 100644 index 000000000..e6d5fb467 --- /dev/null +++ b/scripts/README_release_review.md @@ -0,0 +1,39 @@ +# 本地发布清单与回滚材料 + +这两个脚本只用于准备和检查,不提供上传、远程回滚、迁移或重启命令。 + +在 `D:\WorkProjects\zywlw` 的 PowerShell 中运行: + +```powershell +$env:PYTHONIOENCODING = 'utf-8' +$reviewName = 'outputs/release-review-' + (Get-Date -Format 'yyyyMMdd-HHmmss') +& .\maxkb\.venv\Scripts\python.exe .\maxkb\scripts\release_review.py --output $reviewName +& .\maxkb\.venv\Scripts\python.exe .\maxkb\scripts\release_restore_local.py $reviewName +& .\maxkb\.venv\Scripts\python.exe .\maxkb\scripts\release_restore_local.py $reviewName --destination ($reviewName + '/local-rollback-preview') +``` + +连接复用项目原有私有连接设置和已固定的 SSH 主机密钥;脚本没有复制或输出凭据。新输出目录必须位于本工作区,不能覆盖已有目录。远程只做指定源文件、静态资源哈希及已知进程入口的读取,不调用 Django、业务接口、数据库或缓存。 + +输出内容: + +| 文件 | 用途 | +|---|---| +| README.md | 本轮差异摘要、发布边界与回滚条件 | +| release-plan.json | 源码、构建、远程资源哈希及将来发布顺序 | +| rollback-plan.json | 旧文件哈希、将来新增文件的处理清单 | +| source.diff / static-entry.diff | 源码及两处入口的文本差异 | +| publish-candidate.zip | 核对时刻的本地候选源文件与构建,未发布 | +| rollback-before.zip | 远程旧源文件、旧入口及入口直接引用的 JS/CSS | +| inspection-status.json | 核对是否完成;失败时保留 incomplete 状态 | + +回滚归档不是完整站点或数据库备份。旧 JS 的间接依赖没有全部收录,将来发布必须保留现有静态资源。回滚脚本仅验证归档,或者重建到新的本地目录;不会连接服务器、删除文件或覆盖现有代码。 + +当前脚本中的待办说明针对本次中断范围:单行操作栏、访问记录分页及五类筛选仍待实现;`release_ready` 固定为 false,不能把生成清单视为允许发布。后续功能完成后,应更新范围说明并重新构建、核对和验收。 + +本地安全测试: + +```powershell +& .\maxkb\.venv\Scripts\python.exe .\maxkb\scripts\test_release_review.py -v +``` + +本次实际结果位于 `outputs/release-review-20261026`。 diff --git a/scripts/check_access_control.py b/scripts/check_access_control.py new file mode 100644 index 000000000..afc5d62d6 --- /dev/null +++ b/scripts/check_access_control.py @@ -0,0 +1,31 @@ +"""Isolated in-memory SQLite tests; never touches shared databases, caches or workers.""" +import importlib +import os +from pathlib import Path +import sys +import unittest + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root / 'apps')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'maxkb.settings' +from django.apps import AppConfig +from django.conf import settings +settings.LOGGING_CONFIG = None +settings.USE_I18N = False +settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} +settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in ( + 'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common', + 'system_manage', 'models_provider', 'application', 'chat', 'oss')] +import django +django.setup() +from django.db import connection +from users.models import User +from system_manage.models import SystemSetting, AccessControlPolicy, AccessControlPolicyApplicationRecord, Log +with connection.schema_editor() as editor: + for model in (User, SystemSetting, AccessControlPolicy, AccessControlPolicyApplicationRecord, Log): + editor.create_model(model) +outcome = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromNames([ + 'system_manage.test_access_control', 'system_manage.test_access_control_decisions'])) +sys.exit(0 if outcome.wasSuccessful() else 1) diff --git a/scripts/check_ai_hub.py b/scripts/check_ai_hub.py index 454ef5b9e..4c3d8a0fa 100644 --- a/scripts/check_ai_hub.py +++ b/scripts/check_ai_hub.py @@ -34,5 +34,6 @@ with connection.schema_editor() as editor: for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge, AIParserResource, AIFlow, AIService, AITask, AIArtifact): editor.create_model(model) suite = unittest.defaultTestLoader.loadTestsFromName('system_manage.test_ai_hub') +suite.addTests(unittest.defaultTestLoader.loadTestsFromName('system_manage.test_ai_hub_organize')) outcome = unittest.TextTestRunner(verbosity=2).run(suite) sys.exit(0 if outcome.wasSuccessful() else 1) diff --git a/scripts/release_restore_local.py b/scripts/release_restore_local.py new file mode 100644 index 000000000..e9b46b80b --- /dev/null +++ b/scripts/release_restore_local.py @@ -0,0 +1,75 @@ +"""Verify rollback material, or restore it into a NEW LOCAL directory only.""" +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import sys +import zipfile + +MAX_TOTAL = 128 * 1024 * 1024 + + +def verify(snapshot): + plan = json.loads((snapshot / 'rollback-plan.json').read_text(encoding='utf-8')) + if plan.get('archive') != 'rollback-before.zip': raise ValueError('Unexpected rollback archive') + archive = snapshot / plan['archive'] + if archive.is_symlink(): raise ValueError('Symlink archive refused') + if archive.stat().st_size > MAX_TOTAL: raise ValueError('Archive too large') + if hashlib.sha256(archive.read_bytes()).hexdigest() != plan['archive_sha256']: + raise ValueError('Rollback archive checksum mismatch') + records = plan['files'] + expected = {row['path']: row for row in records} + if len(expected) != len(records): raise ValueError('Duplicate rollback manifest paths') + with zipfile.ZipFile(archive) as zipped: + names = zipped.namelist() + if len(set(names)) != len(names) or set(names) != set(expected): + raise ValueError('Rollback members differ from manifest') + if sum(info.file_size for info in zipped.infolist()) > MAX_TOTAL: raise ValueError('Archive expands beyond limit') + for name in names: + path = PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or '\\' in name or ':' in name: + raise ValueError('Unsafe rollback member') + if not name.startswith(('apps/system_manage/', 'apps/static/admin/', 'ui/src/', 'ui/dist/admin/')): + raise ValueError('Rollback member outside application scope') + if zipped.getinfo(name).file_size != expected[name]['size']: raise ValueError('Member size mismatch') + raw = zipped.read(name) + if hashlib.sha256(raw).hexdigest() != expected[name]['sha256']: + raise ValueError('Rollback file checksum mismatch') + return plan + + +def restore(snapshot, destination): + plan = verify(snapshot) + if destination.exists(): raise ValueError('Destination must be a new local directory') + if any(parent.is_symlink() for parent in [destination, *destination.parents]): + raise ValueError('Symlink destination refused') + destination.mkdir(parents=True, exist_ok=False) + with zipfile.ZipFile(snapshot / plan['archive']) as zipped: + for row in plan['files']: + file = destination / row['path'] + file.parent.mkdir(parents=True, exist_ok=True) + with file.open('xb') as stream: stream.write(zipped.read(row['path'])) + (destination / 'LOCAL_ONLY.txt').write_text( + 'This is a local reconstruction for review. No remote files were restored or removed.\n', encoding='utf-8') + return len(plan['files']) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('snapshot', type=Path) + parser.add_argument('--destination', type=Path, help='Optional new LOCAL reconstruction directory') + args = parser.parse_args() + snapshot = args.snapshot.resolve() + if args.destination: + count = restore(snapshot, args.destination.absolute()) + print(json.dumps({'restored_local_files': count, 'remote_mutation': False})) + else: + plan = verify(snapshot) + print(json.dumps({'verified_files': len(plan['files']), 'remote_mutation': False})) + + +if __name__ == '__main__': + try: main() + except (OSError, ValueError, KeyError, zipfile.BadZipFile) as exc: + print('Local rollback verification failed: ' + type(exc).__name__, file=sys.stderr) + sys.exit(1) diff --git a/scripts/release_review.py b/scripts/release_review.py new file mode 100644 index 000000000..144ea8040 --- /dev/null +++ b/scripts/release_review.py @@ -0,0 +1,308 @@ +"""Prepare LOCAL release/rollback material using read-only SSH inspection. + +There is deliberately no apply, upload, migration, or service-control command. +Run from the workspace with the existing project Python/SSH libraries. +""" +import argparse +from datetime import datetime, timedelta, timezone +import difflib +import errno +import hashlib +import importlib.util +import json +from pathlib import Path, PurePosixPath +import re +import shlex +import stat +import sys +import zipfile + +WORKSPACE = Path(__file__).resolve().parents[2] +PROJECT = WORKSPACE / 'maxkb' +REMOTE_ROOT = '/home/tlw/unis/UnisKB' +HOST = '10.100.52.118' +MAX_FILE = 32 * 1024 * 1024 +SOURCES = [ + 'apps/system_manage/document_export.py', 'apps/system_manage/test_document_export.py', + 'apps/system_manage/views/ai_hub.py', 'apps/system_manage/urls.py', + 'ui/src/api/ai-hub.ts', 'ui/src/views/mindmap/workbench/api/hub.ts', + 'ui/src/views/mindmap/workbench/utils/graphModel.ts', + 'ui/src/views/mindmap/workbench/utils/graphLayout.ts', + 'ui/src/views/mindmap/workbench/utils/exportFilename.ts', + 'ui/src/views/mindmap/workbench/utils/imageAssets.ts', + 'ui/src/views/mindmap/workbench/components/MindMapRenderer.vue', + 'ui/src/views/mindmap/workbench/components/MarkdownRenderer.vue', + 'ui/src/views/mindmap/workbench/views/DocumentProcessor.vue', + 'apps/system_manage/access_control.py', + 'apps/system_manage/views/access_control_policy.py', + 'ui/src/api/system-settings/access-control-policy.ts', + 'ui/src/views/system-setting/policy/index.vue', +] +ENTRIES = ['apps/static/admin/index.html', 'ui/dist/admin/index.html'] +STATIC_ROOTS = ['apps/static/admin', 'ui/dist/admin'] +ASSET_PATTERN = re.compile(r'''(?:src|href)=["']/admin/(assets/[A-Za-z0-9_.-]+)["']''') + +# Fixed code only: metadata, hashes, and recognized process entries. No importing +# the application, reading /proc/*/environ, writing files, or opening a DB/cache. +REMOTE_PROBE = r''' +import hashlib,json,os,stat,sys +from pathlib import Path,PurePosixPath +root=Path('/home/tlw/unis/UnisKB') +payload=json.load(sys.stdin) +if len(payload['paths'])>2000:raise ValueError('Too many paths') +files={} +for relative in payload['paths']: + p=PurePosixPath(relative) + if p.is_absolute() or '..' in p.parts or '\\' in relative:raise ValueError('Invalid path') + allowed=relative.startswith(('apps/system_manage/','ui/src/','apps/static/admin/','ui/dist/admin/')) + if not allowed:raise ValueError('Outside release scope') + path=root/relative + try: + for parent in [path,*path.parents]: + if parent==root:break + if parent.is_symlink():raise ValueError('Symlink target refused') + info=path.stat() + if not stat.S_ISREG(info.st_mode) or info.st_size>33554432:raise ValueError('Unexpected file') + h=hashlib.sha256() + with path.open('rb') as stream: + for chunk in iter(lambda:stream.read(1048576),b''):h.update(chunk) + files[relative]={'exists':True,'sha256':h.hexdigest(),'size':info.st_size} + except FileNotFoundError:files[relative]={'exists':False} +processes=[] +for proc in Path('/proc').iterdir(): + if not proc.name.isdigit():continue + try: + if (proc/'cwd').resolve()!=root:continue + args=[a for a in (proc/'cmdline').read_bytes().decode().split('\0') if a] + known=next((entry for entry in ('scripts/dev_local.py','scripts/dev_local_model.py','ai_hub_worker') if entry in args),None) + if known:processes.append({'pid':int(proc.name),'entry':known,'executable':args[0]}) + except (OSError,UnicodeError):pass +print(json.dumps({'files':files,'processes':processes,'inspection':'read_only'})) +''' + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def safe_relative(value): + path = PurePosixPath(value) + if path.is_absolute() or '..' in path.parts or '\\' in value or ':' in value or not path.parts: + raise ValueError('Unsafe relative path') + return path.as_posix() + + +def compare(path, before, after): + state = 'added' if before is None else 'same' if before == after else 'modified' + delta = list(difflib.unified_diff( + (before or b'').decode('utf-8-sig').splitlines(), + after.decode('utf-8-sig').splitlines(), + fromfile='remote/' + path, tofile='local/' + path, lineterm='')) + return {'path': path, 'status': state, + 'remote_sha256': sha(before) if before is not None else None, + 'local_sha256': sha(after), 'size': len(after), + 'added_lines': sum(line.startswith('+') and not line.startswith('+++') for line in delta), + 'removed_lines': sum(line.startswith('-') and not line.startswith('---') for line in delta), + 'line_endings_only': state == 'modified' and not delta}, delta + + +def read_remote(sftp, relative): + """All SFTP opens are rb; missing files are distinct from denied reads.""" + relative = safe_relative(relative) + for index in range(1, len(PurePosixPath(relative).parts) + 1): + partial = '/'.join(PurePosixPath(relative).parts[:index]) + try: + info = sftp.lstat(REMOTE_ROOT + '/' + partial) + except OSError as exc: + if exc.errno == errno.ENOENT: + return None + raise RuntimeError('Remote metadata read failed: ' + relative) from None + if stat.S_ISLNK(info.st_mode): + raise ValueError('Remote symlink refused: ' + relative) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_FILE: + raise ValueError('Remote file outside size/type limit: ' + relative) + with sftp.open(REMOTE_ROOT + '/' + relative, 'rb') as stream: + raw = stream.read(MAX_FILE + 1) + if len(raw) > MAX_FILE: + raise ValueError('Remote file too large: ' + relative) + return raw + + +def local_file(root, relative): + relative = safe_relative(relative) + file = root / relative + if file.is_symlink() or not file.resolve().is_relative_to(root.resolve()): + raise ValueError('Local symlink or escaped path refused') + if not file.is_file() or file.stat().st_size > MAX_FILE: + raise ValueError('Local file missing or too large: ' + relative) + return file.read_bytes() + + +def connect(): + # Reuse the established secret reader and pinned host keys. Never copy secret + # values into this script, a manifest, command-line arguments, or its outputs. + sys.path.insert(0, str(WORKSPACE / '.codex_tmp/ssh-libs')) + spec = importlib.util.spec_from_file_location('release_connection', WORKSPACE / '.codex_tmp/local-dev-runner.py') + runner = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runner) + if runner.ssh_settings()[0] != HOST: + raise ValueError('Existing connection target does not match the approved host') + return runner.connect() + + +def probe(client, paths): + stdin, stdout, stderr = client.exec_command('python3 -c ' + shlex.quote(REMOTE_PROBE), timeout=60) + stdin.write(json.dumps({'paths': sorted(set(paths))})) + stdin.channel.shutdown_write() + raw = stdout.read() + error = stderr.read() # Consume without echoing arbitrary remote diagnostics. + if stdout.channel.recv_exit_status() or error: + raise RuntimeError('Remote read-only metadata inspection failed') + return json.loads(raw) + + +def save_json(path, value): + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + + +def prepare(output, build, client): + """Only local output is written; remote source/entry drift aborts the snapshot.""" + output.mkdir(parents=True, exist_ok=False) + save_json(output / 'inspection-status.json', {'status': 'incomplete', 'remote_mutation': False}) + sftp = client.open_sftp() + before_files, after_files, source_rows, source_diff, entry_rows, entry_diff = {}, {}, [], [], [], [] + rollback_remove = [] + try: + for relative in SOURCES: + after = local_file(PROJECT, relative) + before = read_remote(sftp, relative) + row, delta = compare(relative, before, after) + source_rows.append(row); source_diff.extend(delta) + if row['status'] != 'same': + after_files['source/' + relative] = after + if before is not None: before_files[relative] = before + else: rollback_remove.append(relative) + html = local_file(build, 'admin.html') + if not ASSET_PATTERN.search(html.decode('utf-8')): + raise ValueError('Build entry does not use /admin/assets/') + for relative in ENTRIES: + before = read_remote(sftp, relative) + row, delta = compare(relative, before, html) + entry_rows.append(row); entry_diff.extend(delta) + if before is not None: + before_files[relative] = before + # Preserve the old entry's primary assets as additional local + # rollback evidence. Transitive old assets must remain on host. + prefix = str(PurePosixPath(relative).parent) + for asset in ASSET_PATTERN.findall(before.decode('utf-8')): + name = prefix + '/' + safe_relative(asset) + if name not in before_files: + content = read_remote(sftp, name) + if content is None: raise ValueError('Existing entry asset missing: ' + name) + before_files[name] = content + else: rollback_remove.append(relative) + build_rows = [] + for file in sorted(build.rglob('*')): + if not file.is_file(): continue + relative = file.relative_to(build).as_posix() + content = local_file(build, relative) + build_rows.append({'path': relative, 'sha256': sha(content), 'size': len(content)}) + remote_paths = list(SOURCES) + ENTRIES + list(before_files) + remote_paths.extend(prefix + '/' + row['path'] for prefix in STATIC_ROOTS for row in build_rows if row['path'] != 'admin.html') + inventory = probe(client, remote_paths) + for row in source_rows + entry_rows: + metadata = inventory['files'][row['path']] + if metadata.get('sha256') != row['remote_sha256']: + raise RuntimeError('Remote file changed during inspection: ' + row['path']) + for name, content in before_files.items(): + if inventory['files'][name].get('sha256') != sha(content): + raise RuntimeError('Rollback source changed during inspection: ' + name) + static_rows = [] + for prefix in STATIC_ROOTS: + counts = {'same': 0, 'added': 0, 'modified': 0} + for row in build_rows: + if row['path'] == 'admin.html': continue + name = prefix + '/' + row['path']; remote = inventory['files'][name] + status = 'added' if not remote['exists'] else 'same' if remote['sha256'] == row['sha256'] else 'modified' + counts[status] += 1 + static_rows.append({'target': name, 'status': status, 'local_sha256': row['sha256'], 'remote_sha256': remote.get('sha256')}) + print(json.dumps({'static_root': prefix, **counts}), flush=True) + finally: + sftp.close() + # Check build provenance without treating an old test log as fresh acceptance. + provenance = WORKSPACE / '.codex_tmp/mindmap-speed/manifest.json' + known = {row['path']: row['sha256'] for row in json.loads(provenance.read_text(encoding='utf-8'))} if provenance.is_file() else {} + mismatches = [row['path'] for row in source_rows if row['path'] in known and row['local_sha256'] != known[row['path']]] + newer_ui = [row['path'] for row in source_rows if row['path'].startswith('ui/') and (PROJECT / row['path']).stat().st_mtime > (build / 'admin.html').stat().st_mtime + 1] + # Capture the candidate now; later working-tree edits must not silently enter it. + with zipfile.ZipFile(output / 'publish-candidate.zip', 'w', zipfile.ZIP_DEFLATED) as zipped: + for name, content in after_files.items(): zipped.writestr(name, content) + for row in build_rows: + content = local_file(build, row['path']) + if sha(content) != row['sha256']: raise RuntimeError('Build changed while packaging') + zipped.writestr('static/' + row['path'], content) + with zipfile.ZipFile(output / 'rollback-before.zip', 'w', zipfile.ZIP_DEFLATED) as zipped: + for name, content in before_files.items(): zipped.writestr(name, content) + snapshots = [{'path': name, 'sha256': sha(content), 'size': len(content)} for name, content in sorted(before_files.items())] + rollback = {'schema': 1, 'host': HOST, 'remote_root': REMOTE_ROOT, + 'archive': 'rollback-before.zip', 'archive_sha256': sha((output / 'rollback-before.zip').read_bytes()), + 'files': snapshots, 'new_paths_to_remove_only_after_future_approval': rollback_remove, + 'preserve_all_existing_static_assets': True, 'restore_database': False, + 'remote_execution_available': False} + plan = {'schema': 1, 'inspection_mode': 'read_only', 'remote_mutation': False, + 'host': HOST, 'remote_root': REMOTE_ROOT, + 'inspected_at': datetime.now(timezone(timedelta(hours=8))).isoformat(), + 'sources': source_rows, 'entries': entry_rows, 'static_assets': static_rows, + 'build': {'directory': str(build), 'files': build_rows, 'provenance_mismatches': mismatches, 'newer_ui_sources': newer_ui}, + 'processes': inventory['processes'], 'database_migration_planned': False, + 'candidate_archive_sha256': sha((output / 'publish-candidate.zip').read_bytes()), + 'pending_requirements': ['导图操作栏单行精简尚未实现', '访问记录分页及五类筛选尚未实现'], + 'release_ready': False, + 'future_order_only': ['复核并完成待办、重新构建和验收', '重新校验远程 SHA256 防止覆盖他人改动', '备份并同步本轮变更源文件', '同步新静态资源并保留所有旧资源', '按后端改动决定是否重启现有 Web', '最后替换 apps/static/admin 和 ui/dist/admin 两处入口', '真实页面回归后决定保留或回滚']} + save_json(output / 'release-plan.json', plan) + save_json(output / 'rollback-plan.json', rollback) + (output / 'source.diff').write_text('\n'.join(source_diff) + '\n', encoding='utf-8') + (output / 'static-entry.diff').write_text('\n'.join(entry_diff) + '\n', encoding='utf-8') + counts = {state: sum(row['status'] == state for row in source_rows) for state in ('added', 'modified', 'same')} + lines = ['# 本地发布准备与远程只读核对', '', + '本轮仅检查远程文件并在本地生成材料,没有上传、远程写入、迁移、服务重启或部署。', '', + f"源文件共 {len(source_rows)} 个:新增 {counts['added']},修改 {counts['modified']},相同 {counts['same']}。", '', + '## 源码差异', '', '| 文件 | 状态 | 新增行 | 删除行 |', '|---|---|---:|---:|'] + lines += [f"| {row['path']} | {row['status']} | {row['added_lines']} | {row['removed_lines']} |" for row in source_rows] + lines += ['', '## 发布边界', '', '当前材料不是可直接发布的完成版本。单行操作栏及访问记录分页筛选尚未实现;必须补齐后重新构建、核对和验收。', + '构建来源哈希不匹配:' + (', '.join(mismatches) or '无(仅覆盖旧构建清单记载的文件)'), + '晚于构建入口的 UI 源码:' + (', '.join(newer_ui) or '无'), '', + '## 回滚材料', '', + 'rollback-before.zip 保存本轮涉及的旧源文件、两处旧入口及入口直接引用的旧 JS/CSS。', + '归档不是整站或数据库备份;JS 的间接依赖不全部收录,因此将来发布必须保留全部现有静态资源。', + '新增源文件没有旧内容,其路径仅列入回滚计划,本脚本不会删除远程文件。', + 'release_restore_local.py 只能验证归档或解包到一个新的本地目录,不能连接服务器、覆盖现有目录或执行回滚。', '', + '## 核对依据', '', 'release-plan.json:逐文件 SHA256、两处静态入口、构建资源与现有进程入口。', + 'source.diff / static-entry.diff:源码与入口逐行差异。', + 'publish-candidate.zip:核对时刻的本地候选源文件和构建,尚未发布。', + 'rollback-plan.json:旧内容 SHA256、回滚条件与新增路径清单。'] + (output / 'README.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + save_json(output / 'inspection-status.json', {'status': 'complete', 'remote_mutation': False, 'source_counts': counts}) + print(json.dumps({'output': str(output), 'source_counts': counts, 'remote_mutation': False, 'release_ready': False}), flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True, help='New LOCAL output directory within the workspace') + parser.add_argument('--build', type=Path, default=PROJECT / 'ui/dist-mindmap-speed') + args = parser.parse_args() + output = args.output.resolve(); build = args.build.resolve() + if not output.is_relative_to(WORKSPACE) or not build.is_relative_to(PROJECT / 'ui'): + parser.error('Output/build must remain inside the workspace/project UI') + if output.exists(): parser.error('Output directory already exists; use a new snapshot directory') + client = connect() + try: prepare(output, build, client) + finally: client.close() + + +if __name__ == '__main__': + try: main() + except Exception as exc: + # Do not echo dependency/connection exceptions that might include secrets. + print('Release review failed (' + type(exc).__name__ + '); no remote mutation was requested.', file=sys.stderr) + sys.exit(1) diff --git a/scripts/test_release_review.py b/scripts/test_release_review.py new file mode 100644 index 000000000..4a7c9837c --- /dev/null +++ b/scripts/test_release_review.py @@ -0,0 +1,115 @@ +"""Release-plan safety and integrity tests; no network or application services.""" +import copy +import errno +import hashlib +import importlib.util +import json +from pathlib import Path +import stat +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import zipfile +import io + + +def module(name): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + '.py')) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + + +review = module('release_review') +rollback = module('release_restore_local') + + +class ReadOnlySFTP: + def __init__(self, files): self.files = files + + def lstat(self, absolute): + relative = absolute.removeprefix(review.REMOTE_ROOT + '/') + if relative in self.files: + return SimpleNamespace(st_mode=stat.S_IFREG, st_size=len(self.files[relative])) + if any(name.startswith(relative + '/') for name in self.files): + return SimpleNamespace(st_mode=stat.S_IFDIR, st_size=0) + raise FileNotFoundError(errno.ENOENT, 'missing') + + def open(self, absolute, mode): + assert mode == 'rb', 'A remote write was attempted' + return io.BytesIO(self.files[absolute.removeprefix(review.REMOTE_ROOT + '/')]) + + def close(self): pass + + +class ReleaseReviewTests(unittest.TestCase): + def test_snapshot_plan_is_read_only_and_can_reconstruct_old_local_files(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); project = root / 'maxkb'; build = project / 'ui/build' + for path, raw in { + 'apps/system_manage/example.py': b'new source\n', + 'ui/src/new.ts': b'new module\n', + 'ui/build/admin.html': b'', + 'ui/build/assets/new.js': b'new static', + }.items(): + file = project / path; file.parent.mkdir(parents=True, exist_ok=True); file.write_bytes(raw) + files = {'apps/system_manage/example.py': b'old source\n'} + for prefix in review.STATIC_ROOTS: + files[prefix + '/index.html'] = b'' + files[prefix + '/assets/old.js'] = b'old static' + original = copy.deepcopy(files) + client = SimpleNamespace(open_sftp=lambda: ReadOnlySFTP(files)) + + def probe(_client, paths): + return {'files': {name: {'exists': True, 'sha256': review.sha(files[name]), 'size': len(files[name])} + if name in files else {'exists': False} for name in paths}, 'processes': []} + + output = root / 'review' + with patch.object(review, 'WORKSPACE', root), patch.object(review, 'PROJECT', project), \ + patch.object(review, 'SOURCES', ['apps/system_manage/example.py', 'ui/src/new.ts']), \ + patch.object(review, 'probe', side_effect=probe): + review.prepare(output, build, client) + self.assertEqual(files, original) + plan = json.loads((output / 'release-plan.json').read_text(encoding='utf-8')) + self.assertFalse(plan['remote_mutation']) + self.assertFalse(plan['release_ready']) + self.assertEqual([row['status'] for row in plan['sources']], ['modified', 'added']) + verified = rollback.verify(output) + self.assertEqual(len(verified['files']), 5) + self.assertIn('ui/src/new.ts', verified['new_paths_to_remove_only_after_future_approval']) + target = root / 'restored' + self.assertEqual(rollback.restore(output, target), 5) + self.assertEqual((target / 'apps/system_manage/example.py').read_bytes(), b'old source\n') + self.assertFalse((target / 'ui/src/new.ts').exists()) + with self.assertRaises(ValueError): rollback.restore(output, target) + (output / 'rollback-before.zip').write_bytes(b'corrupted') + with self.assertRaises(ValueError): rollback.verify(output) + + def test_permission_denied_is_not_treated_as_a_new_remote_file(self): + def denied(_path): raise PermissionError(errno.EACCES, 'denied') + with self.assertRaises(RuntimeError): + review.read_remote(SimpleNamespace(lstat=denied), 'apps/system_manage/example.py') + + def test_path_traversal_is_rejected(self): + for value in ('../secret', '/etc/passwd', 'C:/secret', 'apps/../../secret', 'ui\\src\\file'): + with self.assertRaises(ValueError): review.safe_relative(value) + + def test_normalized_line_endings_are_not_reported_as_logic_changes(self): + row, delta = review.compare('example.py', b'line\r\n', b'line\n') + self.assertEqual(row['status'], 'modified') + self.assertTrue(row['line_endings_only']) + self.assertFalse(delta) + + def test_remote_probe_contains_no_application_import_or_service_mutation(self): + import ast + tree = ast.parse(review.REMOTE_PROBE) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + self.assertNotIn(node.func.attr, {'write', 'write_text', 'write_bytes', 'mkdir', 'unlink', 'rename', 'remove', 'chmod', 'connect', 'kill'}) + if node.func.attr == 'open': + self.assertEqual(node.args[0].value, 'rb') + + +if __name__ == '__main__': + unittest.main() diff --git a/ui/package.json b/ui/package.json index 95eb6c884..87d1e2ee4 100644 --- a/ui/package.json +++ b/ui/package.json @@ -56,7 +56,11 @@ "vue-draggable-plus": "^0.6.0", "vue-i18n": "^11.1.3", "vue-router": "^4.5.0", - "vue3-menus": "^1.1.2" + "vue3-menus": "^1.1.2", + "markmap-lib": "0.18.12", + "markmap-view": "0.18.12", + "markdown-it": "14.1.0", + "dompurify": "3.3.3" }, "devDependencies": { "@tsconfig/node22": "^22.0.1", diff --git a/ui/src/api/ai-hub.ts b/ui/src/api/ai-hub.ts index 875dddb4d..cfa3da5ff 100644 --- a/ui/src/api/ai-hub.ts +++ b/ui/src/api/ai-hub.ts @@ -2,11 +2,32 @@ import { get, post, request } from '@/request/index' import useStore from '@/stores' export const hubBase = () => `/workspace/${useStore().user.getWorkspaceId()}/ai_hub` -export const hubGet = (kind: string, id?: string) => get(`${hubBase()}/${kind}${id ? '/' + id : ''}`) -export const hubPost = (kind: string, data: any, id?: string) => post(`${hubBase()}/${kind}${id ? '/' + id : ''}`, data) -export async function hubBinary(kind: 'assets' | 'exports', id?: string, data?: any) { - const response = await request({ url: `${hubBase()}/${kind}${id ? '/' + id + '/content' : ''}`, - method: kind === 'assets' ? 'get' : 'post', data, responseType: 'blob' }) - if (!(response.data instanceof Blob) || response.data.type.includes('json')) throw new Error('图片或导出文件不可用') +export const hubGet = (kind: string, id?: string) => + get(`${hubBase()}/${kind}${id ? '/' + id : ''}`) +export const hubPost = (kind: string, data: any, id?: string) => + post(`${hubBase()}/${kind}${id ? '/' + id : ''}`, data) +export const hubDelete = (kind: string, id: string) => + request({ url: `${hubBase()}/${kind}/${id}`, method: 'delete' }) +export async function hubBinary(kind: 'assets' | 'exports' | 'document_export', id?: string, data?: any) { + if (kind === 'document_export') { + // A multipart file bypasses Django's small JSON-body memory limit for image-rich documents. + const form = new FormData() + form.append('content', new Blob([data.html], { type:'text/html;charset=utf-8' }), 'document.html') + form.append('format', data.format); form.append('paper', data.paper || 'A4') + data = form + } + const response = await request({ + url: `${hubBase()}/${kind}${id ? '/' + id + '/content' : ''}`, + method: kind === 'assets' ? 'get' : 'post', + data, + responseType: 'blob', + timeout: kind === 'document_export' ? 120000 : 60000, + }) + if (!(response.data instanceof Blob)) throw new Error('图片或导出文件不可用') + if (response.data.type.includes('json')) { + let message = '图片或导出文件不可用' + try { const detail = JSON.parse(await response.data.text()); if (typeof detail.message === 'string') message = detail.message.slice(0, 200) } catch { /* retain a safe fallback */ } + throw new Error(message) + } return { content: await response.data.arrayBuffer(), mime: response.data.type } } diff --git a/ui/src/api/system-settings/access-control-policy.ts b/ui/src/api/system-settings/access-control-policy.ts index 081a60056..884c2e637 100644 --- a/ui/src/api/system-settings/access-control-policy.ts +++ b/ui/src/api/system-settings/access-control-policy.ts @@ -40,4 +40,8 @@ export default { deletePolicy, applyPolicy, getApplicationRecordList, + getRuntime: () => get(`${prefix}/runtime`), + saveRegions: (region_networks: any[]) => put(`${prefix}/runtime`, { region_networks }), + unbind: (data: any) => del(`${prefix}/runtime`, undefined, data), + getDecisions: (params?: any) => get(`${prefix}/decisions`, params), } diff --git a/ui/src/api/system/platform-access.ts b/ui/src/api/system/platform-access.ts new file mode 100644 index 000000000..45039cfb5 --- /dev/null +++ b/ui/src/api/system/platform-access.ts @@ -0,0 +1,11 @@ +import { get, post, put, del } from '@/request/index' +const prefix = '/platform_access' +export default { + config: () => get(prefix), + saveRole: (data: any, id?: string) => post(`${prefix}/roles${id ? '/' + id : ''}`, data), + deleteRole: (id: string) => del(`${prefix}/roles/${id}`, undefined, {}), + saveGrants: (grants: any[]) => post(`${prefix}/grants`, { grants }), + revokeGrant: (id: string) => del(`${prefix}/grants/${id}`, undefined, {}), + user: (id: string) => get(`${prefix}/users/${id}`), + setUserRoles: (id: string, role_ids: string[]) => put(`${prefix}/users/${id}`, { role_ids }), +} diff --git a/ui/src/layout/components/sidebar/index.vue b/ui/src/layout/components/sidebar/index.vue index 1ce6c6ec6..6b078115a 100644 --- a/ui/src/layout/components/sidebar/index.vue +++ b/ui/src/layout/components/sidebar/index.vue @@ -1,120 +1,183 @@ @@ -68,61 +203,221 @@ - + diff --git a/ui/src/views/mindmap/index.vue b/ui/src/views/mindmap/index.vue index 469e85d70..aac23879a 100644 --- a/ui/src/views/mindmap/index.vue +++ b/ui/src/views/mindmap/index.vue @@ -1,105 +1,4 @@ - - + - - diff --git a/ui/src/views/mindmap/workbench/api/document.ts b/ui/src/views/mindmap/workbench/api/document.ts new file mode 100644 index 000000000..d5e479831 --- /dev/null +++ b/ui/src/views/mindmap/workbench/api/document.ts @@ -0,0 +1,145 @@ +import { hubCall } from './hub' +import { uuid } from '@/views/mindmap/workbench/utils/uuid' +export interface ParseParams { + files: File[] + parser_resource_id: string + output_dir: string + lang_list: string + backend: string + parse_method: string + formula_enable: boolean + table_enable: boolean + server_url?: string + start_page_id: number + end_page_id: number + return_md: boolean + return_middle_json: boolean + response_format_zip: boolean +} + +export interface ParseResult { + backend: string + version: string + results: Record< + string, + { + md_content?: string + middle_json?: string + model_output?: string + content_list?: string + images?: Record + } + > +} + +export interface ParseProgress { + progress: number | null + stage: string + status: 'pending' | 'processing' | 'completed' | 'failed' + error: string | null + file_names: string +} + +export interface MindmapOrganizeProgress extends ParseProgress { + result_md?: string | null +} + +const ids = new Map() +const states: Record = { + PENDING: 'pending', + RUNNING: 'processing', + SUCCESS: 'completed', + FAILED: 'failed', +} +async function progress(taskId: string): Promise { + const id = ids.get(taskId) + if (!id) + return { progress: null, status: 'pending', stage: '准备中', error: null, file_names: '' } + const task = await hubCall('tasks', 'get', undefined, id) + return { + progress: task.status === 'SUCCESS' ? 100 : (task.progress?.percent ?? null), + status: states[task.status], + stage: + task.progress?.stage || + (task.status === 'PENDING' + ? '等待处理' + : task.status === 'RUNNING' + ? '处理中' + : task.status === 'SUCCESS' + ? '完成' + : '失败'), + error: task.error, + file_names: task.name, + result_md: task.result?.markdown, + } +} +export const documentApi = { + async parseDocument(params: ParseParams, taskId: string = uuid()): Promise { + const results: ParseResult['results'] = {} + for (const file of params.files) { + const native = ['doc', 'docx', 'xls', 'xlsx', 'csv', 'html', 'htm', 'md', 'txt'].includes( + file.name.split('.').pop()?.toLowerCase() || '', + ) + if ( + params.parser_resource_id === 'builtin' && + !native && + !file.name.toLowerCase().endsWith('.pdf') + ) + throw new Error('内置解析不支持图片,请选择 MinerU 解析服务') + if (!native && !params.parser_resource_id) throw new Error('请选择 PDF/图片解析服务') + const node = + native || params.parser_resource_id === 'builtin' + ? { type: 'parse', processor: 'builtin' } + : { type: 'parse', resource_id: params.parser_resource_id } + const task = await hubCall( + 'tasks', + 'post', + { name: file.name.slice(0, 128), nodes: [node, { type: 'render' }] }, + undefined, + file, + ) + ids.set(taskId, task.id) + const until = Date.now() + 30 * 60 * 1000 + let completed = false + while (Date.now() < until) { + const current = await hubCall('tasks', 'get', undefined, task.id) + if (current.status === 'FAILED') throw new Error(current.error || '解析失败') + if (current.status === 'SUCCESS') { + results[file.name] = { + md_content: current.result.markdown, + middle_json: JSON.stringify(current.result.source), + } + completed = true + break + } + await new Promise((resolve) => setTimeout(resolve, 1500)) + } + if (!completed) throw new Error('任务仍在处理中,请到管理页查看') + } + return { backend: 'shared', version: '1', results } + }, + async createParseTask(_taskId: string): Promise { + return { progress: null, stage: '准备中', status: 'pending', error: null, file_names: '' } + }, + getParseProgress: progress, + async createMindmapTask( + taskId: string, + markdown: string, + _mode = 'smart', + prompt?: string, + config?: { model_id: string; temperature?: number | null }, + ): Promise { + if (!config?.model_id) throw new Error('请选择整理模型') + const node: Record = { type: 'organize', model_id: config.model_id, prompt } + if (config.temperature !== null && config.temperature !== undefined) + node.temperature = config.temperature + const task = await hubCall('tasks', 'post', { + name: '智能整理', + markdown, + nodes: [node, { type: 'render' }], + }) + ids.set(taskId, task.id) + return progress(taskId) + }, + getMindmapProgress: progress, +} diff --git a/ui/src/views/mindmap/workbench/api/hub.ts b/ui/src/views/mindmap/workbench/api/hub.ts new file mode 100644 index 000000000..f63d294b8 --- /dev/null +++ b/ui/src/views/mindmap/workbench/api/hub.ts @@ -0,0 +1,24 @@ +import { watch } from 'vue' +import { useRoute } from 'vue-router' +import { ElMessage } from 'element-plus' +import { hubGet, hubPost, hubBinary } from '@/api/ai-hub' + +export async function hubCall(kind: string, method = 'get', data?: any, id?: string, file?: File): Promise { + if (kind === 'assets' || kind === 'exports' || kind === 'document_export') return hubBinary(kind, id, data) + if (file) { + const form = new FormData() + form.append('data', JSON.stringify(data)); form.append('file', file) + data = form + } + const response = method === 'get' ? await hubGet(kind, id) : await hubPost(kind, data, id) + return response.data +} + +export function onOpenArtifact(callback: (artifact: any) => void) { + const route = useRoute() + return watch(() => route.query.artifact, async id => { + if (typeof id !== 'string' || !/^[a-f0-9-]{36}$/i.test(id)) return + try { callback(await hubCall('artifacts', 'get', undefined, id)) } + catch { ElMessage.error('导图不存在或无权访问') } + }, { immediate: true }) +} diff --git a/ui/src/views/mindmap/workbench/components/ConfigPanel.vue b/ui/src/views/mindmap/workbench/components/ConfigPanel.vue new file mode 100644 index 000000000..4bcac4c0e --- /dev/null +++ b/ui/src/views/mindmap/workbench/components/ConfigPanel.vue @@ -0,0 +1,101 @@ + + + diff --git a/ui/src/views/mindmap/workbench/components/MarkdownRenderer.vue b/ui/src/views/mindmap/workbench/components/MarkdownRenderer.vue new file mode 100644 index 000000000..90a38934d --- /dev/null +++ b/ui/src/views/mindmap/workbench/components/MarkdownRenderer.vue @@ -0,0 +1,588 @@ + + + + + diff --git a/ui/src/views/mindmap/workbench/components/MindMapRenderer.vue b/ui/src/views/mindmap/workbench/components/MindMapRenderer.vue new file mode 100644 index 000000000..c767756f0 --- /dev/null +++ b/ui/src/views/mindmap/workbench/components/MindMapRenderer.vue @@ -0,0 +1,970 @@ + + + + + diff --git a/ui/src/views/mindmap/workbench/composables/useDocumentProcessor.ts b/ui/src/views/mindmap/workbench/composables/useDocumentProcessor.ts new file mode 100644 index 000000000..34e3f8f46 --- /dev/null +++ b/ui/src/views/mindmap/workbench/composables/useDocumentProcessor.ts @@ -0,0 +1,412 @@ +import { ref, reactive, watch } from 'vue' +import type { Ref } from 'vue' +import { documentApi, type ParseParams } from '@/views/mindmap/workbench/api/document' +import { buildMindmapMarkdown, documentTitleFromName } from '@/views/mindmap/workbench/utils/mindmapMarkdown' + +// 根据上一级标题自动补全下一级标题 +export function autoPromoteParagraphsToSubheading(text: string): string { + return buildMindmapMarkdown(text) +} + +export interface DocumentConfig { + maxPages: number + backend: string + serverUrl: string + exportResolution: number + mindmapPrompt: string + parserResourceId: string + mindmapModelId: string + mindmapLlmTemperature: number | null + tableEnable: boolean + formulaEnable: boolean + language: string + forceOcr: boolean +} + +export interface ProcessResult { + markdown: string + source: string + mindmap: string + downloadUrl?: string +} + +export const DOCUMENT_CONFIG_STORAGE_KEY = 'mineru.documentProcessor.config.v5' + +export const DEFAULT_MINDMAP_PROMPT = `你是文档结构整理助手。请基于用户提供的 Markdown 生成适合思维导图展示的 Markdown。 + +要求: +1. 保留原文标题结构,不要重写或打乱主要标题层级。 +2. 将标题下的段落内容总结为要点,合并相近段落,避免逐段照抄。 +3. 保留原文语言:英文内容输出英文,中文内容输出中文,多语言内容按原文语言分别保留。 +4. 不要编造原文没有的信息。 +5. 保留关键数字、公式、专有名词、步骤和结论。 +6. 最大层级不超过 4 层。 +7. 每个父节点下最多 8 个子节点。 +8. 节点标题尽量简短,正文说明使用短句列表。 +9. 只输出 Markdown,不要输出解释、代码块围栏或额外说明。` + +export const DEFAULT_DOCUMENT_CONFIG: DocumentConfig = { + maxPages: 1000, + backend: 'hybrid-auto-engine', + serverUrl: 'http://localhost:30000', + exportResolution: 4096, + mindmapPrompt: DEFAULT_MINDMAP_PROMPT, + parserResourceId: '', + mindmapModelId: '', + mindmapLlmTemperature: null, + tableEnable: true, + formulaEnable: true, + language: 'ch', + forceOcr: false +} + +function loadCachedConfig(): Partial { + try { + const raw = window.localStorage.getItem(DOCUMENT_CONFIG_STORAGE_KEY) + const cached = raw ? JSON.parse(raw) : {} + const safe = Object.fromEntries(Object.keys(DEFAULT_DOCUMENT_CONFIG).filter(key => key in cached).map(key => [key, cached[key]])) + if ('exportResolution' in safe) { + const legacySizes: Record = { 1080: 1024, 2560: 2048, 3840: 4096, 7680: 8192, 16384: 8192 } + const size = Number(safe.exportResolution) + safe.exportResolution = legacySizes[size] ?? ([1024, 2048, 4096, 8192].includes(size) ? size : 4096) + } + window.localStorage.setItem(DOCUMENT_CONFIG_STORAGE_KEY, JSON.stringify(safe)) + return safe + } catch { + try { window.localStorage.removeItem(DOCUMENT_CONFIG_STORAGE_KEY) } catch { /* Storage unavailable. */ } + return {} + } +} + +export function useDocumentProcessor() { + // 文件相关 + const uploadedFiles: Ref = ref([]) + const uploadedDisplayFiles: Ref = ref([]) + const isUploading = ref(false) + + // 配置相关 + const config = reactive({ + ...DEFAULT_DOCUMENT_CONFIG, + ...loadCachedConfig() + }) + + watch( + config, + (value) => { + try { + window.localStorage.setItem(DOCUMENT_CONFIG_STORAGE_KEY, JSON.stringify(value)) + } catch { + // 浏览器禁用本地存储时不影响当前页面使用。 + } + }, + { deep: true } + ) + + // 结果相关 + const results = ref(null) + const isProcessing = ref(false) + const processingStage = ref('') + const error = ref(null) + + // 后端选项 + const backendOptions = [ + { value: 'pipeline', label: '传统管道解析' }, + { value: 'vlm-auto-engine', label: 'VLM本地引擎' }, + { value: 'hybrid-auto-engine', label: '混合本地引擎' }, + { value: 'vlm-http-client', label: 'VLM远程客户端' }, + { value: 'hybrid-http-client', label: '混合远程客户端' } + ] + + // 语言选项 + const languageOptions = [ + { value: 'ch', label: '中文(简体)' }, + { value: 'en', label: '英语' }, + { value: 'korean', label: '韩语' }, + { value: 'japan', label: '日语' }, + { value: 'chinese_cht', label: '中文(繁体)' }, + { value: 'ta', label: '泰米尔语' }, + { value: 'te', label: '泰卢固语' }, + { value: 'ka', label: '卡纳达语' }, + { value: 'th', label: '泰语' }, + { value: 'el', label: '希腊语' }, + { value: 'latin', label: '拉丁语系' }, + { value: 'arabic', label: '阿拉伯语系' }, + { value: 'east_slavic', label: '东斯拉夫语系' }, + { value: 'cyrillic', label: '西里尔语系' }, + { value: 'devanagari', label: '梵文字母语系' } + ] + + // 文件上传处理 + const handleFileUpload = async (files: FileList | null) => { + console.log('handleFileUpload called with files:', files) + if (!files || files.length === 0) return + + const validFiles: File[] = [] + const displayFiles: File[] = [] + const file = files[0] + uploadedFiles.value = [] + uploadedDisplayFiles.value = [] + results.value = null + error.value = null + if (files.length > 1) { + error.value = '一次只能选择一个文件,已使用第一个文件' + } + + const fileType = file.type + const fileName = file.name.toLowerCase() + + console.log('Processing file:', fileName, 'type:', fileType) + + const extension = fileName.split('.').pop() || '' + const isMarkdown = ['md', 'txt'].includes(extension) + if (!['md', 'txt', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'csv', 'html', 'htm', + 'png', 'jpg', 'jpeg', 'webp', 'bmp', 'tiff', 'gif', 'jp2'].includes(extension)) { + error.value = '不支持的文件格式' + return + } + // 验证文件大小 (100MB) + if (file.size > 20 * 1024 * 1024) { + error.value = '文件不能超过20MB' + console.log('File too large:', fileName, file.size) + return + } + + if (isMarkdown) { + try { + isUploading.value = true + const markdown = await file.text() + validFiles.push(file) + displayFiles.push(file) + results.value = { + markdown, + source: markdown, + mindmap: buildMindmapMarkdown(markdown, documentTitleFromName(file.name)) + } + error.value = null + console.log('Markdown file loaded directly:', fileName, markdown.length) + } catch (err) { + error.value = 'Markdown 文件读取失败: ' + (err as Error).message + console.error('Markdown read failed:', (err as Error).message) + return + } finally { + isUploading.value = false + } + } else { + validFiles.push(file) + displayFiles.push(file) + console.log('Adding file directly:', fileName) + } + + uploadedFiles.value = validFiles + uploadedDisplayFiles.value = displayFiles + console.log('Final uploaded files:', validFiles.map(f => f.name)) + } + + // 清除所有数据 + const clearAll = () => { + uploadedFiles.value = [] + uploadedDisplayFiles.value = [] + results.value = null + error.value = null + processingStage.value = '' + } + + // 演示模式:上传文件后由用户手动粘贴 Markdown 源码 + const initializeManualResult = () => { + results.value = { + markdown: '', + source: '', + mindmap: '' + } + error.value = null + } + + // 进度相关 + const progressPercent = ref(0) + const progressStage = ref('') + let progressTimer: ReturnType | null = null + let progressGeneration = 0 + + // 启动进度轮询 + const startProgressPolling = (taskId: string) => { + stopProgressPolling() + const generation = progressGeneration + let polling = false + progressPercent.value = 5 + progressStage.value = '准备解析' + progressTimer = setInterval(async () => { + if (polling) return + polling = true + try { + const data = await documentApi.getParseProgress(taskId) + if (generation !== progressGeneration) return + const stage = data.stage || '' + let estimate = 20 + if (data.status === 'completed' || /生成|渲染/.test(stage)) { + progressStage.value = '生成思维导图' + estimate = 95 + } else if (data.status === 'failed') { + progressStage.value = '解析失败' + estimate = progressPercent.value + } else if (data.status === 'pending' || /等待|排队/.test(stage)) { + progressStage.value = '等待处理' + } else if (/整理|获取.*结果/.test(stage)) { + progressStage.value = '整理解析结果' + estimate = 90 + } else if (/上传|提交/.test(stage)) { + progressStage.value = '提交文档' + } else { + progressStage.value = '解析文档' + // This is workflow progress, not the parser's page completion rate. + // Move gradually within this phase and reserve completion for success. + estimate = data.progress === null + ? Math.min(85, Math.max(35, progressPercent.value + Math.max(1, Math.ceil((85 - progressPercent.value) / 12)))) + : Math.min(85, 35 + Math.round(data.progress * 0.5)) + } + progressPercent.value = Math.max(progressPercent.value, estimate) + if (data.status === 'completed' || data.status === 'failed') { + stopProgressPolling() + } + } catch { + // 忽略轮询失败 + } finally { + polling = false + } + }, 1000) + } + + // 停止进度轮询 + const stopProgressPolling = () => { + progressGeneration += 1 + if (progressTimer) { + clearInterval(progressTimer) + progressTimer = null + } + } + + // 处理文档转换 + const processDocument = async () => { + if (uploadedFiles.value.length === 0) { + error.value = '请先上传文件' + return + } + + const firstFileName = uploadedFiles.value[0]?.name.toLowerCase() || '' + if (firstFileName.endsWith('.md')) { + if (!results.value) { + error.value = 'Markdown 文件内容为空,请重新上传' + } + return + } + + isProcessing.value = true + error.value = null + processingStage.value = '准备提交解析任务' + + // 生成任务ID + const taskId = `task-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + + try { + await documentApi.createParseTask(taskId) + startProgressPolling(taskId) + processingStage.value = '提交文档到解析服务' + progressStage.value = '提交文档' + progressPercent.value = 15 + const params: ParseParams = { + files: uploadedFiles.value, + parser_resource_id: config.parserResourceId, + output_dir: './output', + lang_list: config.language, + backend: config.backend, + parse_method: config.forceOcr ? 'ocr' : 'auto', + formula_enable: config.formulaEnable, + table_enable: config.tableEnable, + start_page_id: 0, + end_page_id: config.maxPages - 1, + return_md: true, + return_middle_json: false, + response_format_zip: false + } + + if (config.backend.includes('http-client') && config.serverUrl) { + params.server_url = config.serverUrl + } + + processingStage.value = '解析文档' + const response = await documentApi.parseDocument(params, taskId) + + if (response.results) { + stopProgressPolling() + progressPercent.value = 95 + progressStage.value = '生成思维导图' + processingStage.value = '生成 Markdown 和思维导图' + const resultData = Object.values(response.results)[0] + const mdContent = resultData.md_content || '' + const mindmapContent = buildMindmapMarkdown(mdContent, documentTitleFromName(uploadedDisplayFiles.value[0]?.name || '')) + results.value = { + markdown: mdContent, + source: mdContent, + mindmap: mindmapContent + } + progressPercent.value = 100 + } + + } catch (err: any) { + error.value = err.message || '转换失败' + } finally { + stopProgressPolling() + processingStage.value = '' + isProcessing.value = false + } + } + + // 根据后端类型获取公式标签 + const getFormulaLabel = (backend: string) => { + if (backend.startsWith('vlm')) { + return '启用行间公式识别' + } else if (backend === 'pipeline') { + return '启用公式识别' + } else if (backend.startsWith('hybrid')) { + return '启用行内公式识别' + } + return '启用公式识别' + } + + // 根据后端类型获取公式说明 + const getFormulaInfo = (backend: string) => { + if (backend.startsWith('vlm')) { + return '禁用后,行间公式将显示为图片。' + } else if (backend === 'pipeline') { + return '禁用后,行间公式将显示为图片,行内公式将不会被检测或解析。' + } else if (backend.startsWith('hybrid')) { + return '禁用后,行内公式将不会被检测或解析。' + } + return '' + } + + return { + // 数据 + uploadedFiles, + uploadedDisplayFiles, + config, + results, + isUploading, + isProcessing, + processingStage, + progressPercent, + progressStage, + error, + + // 选项 + backendOptions, + languageOptions, + + // 方法 + handleFileUpload, + initializeManualResult, + clearAll, + processDocument, + getFormulaLabel, + getFormulaInfo + } +} diff --git a/ui/src/views/mindmap/workbench/utils/exportFilename.ts b/ui/src/views/mindmap/workbench/utils/exportFilename.ts new file mode 100644 index 000000000..f25d9f533 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/exportFilename.ts @@ -0,0 +1,19 @@ +// A source-first name shared by every workbench download. Keep Chinese and dots +// in versioned names, remove only known file extensions and unsafe path characters. +const extensions = /\.(?:pdf|docx?|xlsx?|csv|html?|md|txt|png|jpe?g|webp|bmp|tiff?|gif|jp2|rtf|svg|mmd|zip)$/i +let previous = '', sequence = 0 +export function sourceStem(name: string): string { + const leaf = (name || '').split(/[\\/]/).pop() || '未命名文档' + let stem = leaf.replace(extensions, '').replace(/[<>:"/\\|?*\u0000-\u001f]/g, '_').replace(/[.\s]+$/g, '').trim() + if (!stem) stem = '未命名文档' + if (/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(stem)) stem = '_' + stem + return Array.from(stem).slice(0, 80).join('') +} +export function exportFilename(source: string, parts: string[], extension: string, now = new Date()): string { + const pad = (n: number) => String(n).padStart(2, '0') + const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}` + const base = [sourceStem(source), ...parts, stamp].join('_') + sequence = base === previous ? sequence + 1 : 0 + previous = base + return `${base}${sequence ? '_' + String(sequence + 1).padStart(2, '0') : ''}.${extension}` +} diff --git a/ui/src/views/mindmap/workbench/utils/graphLayout.ts b/ui/src/views/mindmap/workbench/utils/graphLayout.ts new file mode 100644 index 000000000..1d6fd65a6 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/graphLayout.ts @@ -0,0 +1,35 @@ +import type { GraphNode, GraphLayout } from './graphModel' +export interface Positioned { node: GraphNode; x: number; y: number; width: number; height: number } +export function layoutGraph(root: GraphNode, mode: GraphLayout, sizes: Map): Positioned[] { + const positioned: Positioned[] = [], gap = 34, levelGap = 100 + const size = (node: GraphNode) => sizes.get(node.id) || { width: 180, height: 40 } + const children = (node: GraphNode) => node.payload.fold ? [] : node.children + const spans = new Map(), levels: number[] = [] + function measure(node: GraphNode, depth: number): number { + const s = size(node), vertical = mode === 'vertical' + levels[depth] = Math.max(levels[depth] || 0, vertical ? s.height : s.width) + const values = children(node).map(child => measure(child, depth + 1)) + const span = Math.max(mode === 'radial' ? Math.max(s.width, s.height) : vertical ? s.width : s.height, values.reduce((a, b) => a + b, 0) + Math.max(0, values.length - 1) * gap) + spans.set(node.id, span); return span + } + const total = measure(root, 0), axes = [0] + levels.forEach((width, i) => axes.push(axes[i] + width / 2 + (levels[i + 1] || 0) / 2 + levelGap)) + function place(node: GraphNode, depth: number, offset: number) { + const span = spans.get(node.id)!, s = size(node) + const cross = offset + span / 2 - total / 2 + let x = axes[depth], y = cross + if (mode === 'vertical') { x = cross; y = axes[depth] } + if (mode === 'radial' && depth) { + const angle = (cross / total) * Math.PI * 2 + // Circumference grows with the real measured subtree footprint. + const radius = axes[depth] + total / (2 * Math.PI) * 1.1 + x = Math.cos(angle) * radius; y = Math.sin(angle) * radius + } + positioned.push({ node, x, y, ...s }) + const kids = children(node), used = kids.reduce((a, c) => a + spans.get(c.id)!, 0) + Math.max(0, kids.length - 1) * gap + let cursor = offset + (span - used) / 2 + kids.forEach(child => { place(child, depth + 1, cursor); cursor += spans.get(child.id)! + gap }) + } + place(root, 0, 0) + return positioned +} diff --git a/ui/src/views/mindmap/workbench/utils/graphModel.ts b/ui/src/views/mindmap/workbench/utils/graphModel.ts new file mode 100644 index 000000000..3603a6216 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/graphModel.ts @@ -0,0 +1,97 @@ +import MarkdownIt from 'markdown-it' +import katex from 'katex' +import { sanitizeHtml } from './sanitize' + +export type GraphLayout = 'tree' | 'radial' | 'horizontal' | 'vertical' +export interface GraphNode { + id: string + content: string + text: string + children: GraphNode[] + payload: { fold?: number; source?: { start: number; end: number; original: string }; [key: string]: any } +} +export const markdownEngine = new MarkdownIt({ html: true, breaks: true, linkify: true }) +// Parse math before ordinary text, while keeping escaped dollar signs and code intact. +markdownEngine.inline.ruler.before('escape', 'workbench_math', (state: any, silent: boolean) => { + if (state.src[state.pos] !== '$') return false + const delimiter = state.src.startsWith('$$', state.pos) ? '$$' : '$' + const end = state.src.indexOf(delimiter, state.pos + delimiter.length) + if (end < 0 || end === state.pos + delimiter.length) return false + if (!silent) { + const token = state.push('workbench_math', '', 0) + token.content = state.src.slice(state.pos + delimiter.length, end) + token.block = delimiter.length === 2 + } + state.pos = end + delimiter.length + return true +}) +markdownEngine.renderer.rules.workbench_math = (tokens: any, index: number) => { + const token = tokens[index] + return `${katex.renderToString(token.content, { throwOnError: false, displayMode: token.block, trust: false })}` +} + +export function graphFromMarkdown(source: string, title: string): GraphNode { + const root: GraphNode = { id: 'root', text: title, content: markdownEngine.utils.escapeHtml(title), children: [], payload: {} } + const lines = source.split('\n'), offsets: number[] = [0] + lines.forEach(line => offsets.push(offsets[offsets.length - 1] + line.length + 1)) + const tokens = markdownEngine.parse(source, {}) + const headings: { level: number; node: GraphNode }[] = [{ level: 0, node: root }] + const items: (GraphNode | undefined)[] = [] + let depth = 0 + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i] + if (['bullet_list_open', 'ordered_list_open'].includes(token.type)) { depth++; continue } + if (['bullet_list_close', 'ordered_list_close'].includes(token.type)) { items.length = --depth; continue } + if (token.type === 'list_item_open') { items[depth - 1] = undefined; continue } + const heading = token.type === 'heading_open' + if (!heading && !['paragraph_open', 'fence', 'code_block', 'table_open', 'html_block'].includes(token.type)) continue + const map = token.map + if (!map) continue + const start = offsets[map[0]], end = Math.min(source.length, offsets[map[1]] - 1) + const raw = source.slice(start, end) + const inline = tokens[i + 1]?.type === 'inline' ? tokens[i + 1].content : token.content + const isBlock = ['fence', 'code_block', 'table_open', 'html_block'].includes(token.type) + const text = isBlock ? (token.type === 'table_open' ? raw : token.content.trim()) : inline + const node: GraphNode = { id: `${token.type}:${map[0]}:${map[1]}`, text, content: sanitizeHtml(isBlock ? markdownEngine.render(raw) : markdownEngine.renderInline(text)), children: [], payload: { source: { start, end, original: raw } } } + if (!isBlock && map[1] === map[0] + 1) { + // Exact offsets distinguish identical labels in different sections. + const relative = raw.indexOf(inline) + if (relative >= 0) node.payload.source = { start: start + relative, end: start + relative + inline.length, original: inline } + } + if (heading) { + const level = Number(token.tag.slice(1)) + while (headings.length > 1 && headings[headings.length - 1].level >= level) headings.pop() + headings[headings.length - 1].node.children.push(node) + headings.push({ level, node }); items.length = 0 + } else { + const parent = depth > 1 ? items[depth - 2] : undefined + ;(parent || headings[headings.length - 1].node).children.push(node) + if (depth) items[depth - 1] = node + } + if (token.type === 'table_open') while (i < tokens.length && tokens[i].type !== 'table_close') i++ + } + return root +} + +export function walkGraph(root: GraphNode): GraphNode[] { return [root, ...root.children.flatMap(walkGraph)] } +export function graphText(node: GraphNode): string { + const element = document.createElement('div'); element.innerHTML = node.content + return (element.textContent || node.text).replace(/\s+/g, ' ').trim() +} +export function projectGraph(root: GraphNode, query = '', maxDepth = 0, branch = '', matchesOnly = false): GraphNode { + const keyword = query.trim().toLocaleLowerCase() + const visit = (node: GraphNode, level: number, inside: boolean): GraphNode | undefined => { + inside = inside || node.id === branch + const children = maxDepth && level >= maxDepth ? [] : node.children.map(n => visit(n, level + 1, inside)).filter(Boolean) as GraphNode[] + const match = !keyword || graphText(node).toLocaleLowerCase().includes(keyword) + if (node.id !== 'root' && ((branch && !inside && !children.length) || (matchesOnly && keyword && !match && !children.length))) return + return { ...node, payload: { ...node.payload, fold: matchesOnly || branch ? 0 : node.payload.fold }, children } + } + return visit(root, 0, false)! +} +export function graphToMermaid(root: GraphNode, vertical = false): string { + const nodes = walkGraph(root), ids = new Map(nodes.map((n, i) => [n.id, `n${i}`])) + // flowchart supports explicit, collision-free IDs and arbitrary Unicode labels. + const label = (text: string) => text.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>').replace(/[\r\n]+/g, ' ') + return [`flowchart ${vertical ? 'TD' : 'LR'}`, ...nodes.map(n => ` ${ids.get(n.id)}["${label(graphText(n))}"]`), ...nodes.flatMap(n => n.children.map(c => ` ${ids.get(n.id)} --> ${ids.get(c.id)}`))].join('\n') + '\n' +} diff --git a/ui/src/views/mindmap/workbench/utils/imageAssets.ts b/ui/src/views/mindmap/workbench/utils/imageAssets.ts new file mode 100644 index 000000000..6da93143d --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/imageAssets.ts @@ -0,0 +1,100 @@ +import { onBeforeUnmount, ref, watch, type Ref } from 'vue' +import { hubCall } from '@/views/mindmap/workbench/api/hub' + +const pattern = () => /\/admin\/api\/workspace\/[^/\s"<>]+\/ai_hub\/assets\/([a-f0-9-]{36})\/content/g +const cache = new Map() +const pending = new Map>() +let cachedBytes = 0 + +async function imageBlob(url: string, id: string) { + if (cache.has(url)) return cache.get(url)! + if (pending.has(url)) return pending.get(url)! + const request = hubCall('assets', 'get', undefined, id).then(result => { + if (!(result.content instanceof ArrayBuffer) || !/^image\/(png|jpeg|gif|webp|bmp)$/.test(result.mime)) throw new Error('图片加载失败') + const blob = new Blob([result.content], { type: result.mime }) + while (cache.size && cachedBytes + blob.size > 64 * 1024 * 1024) { + const oldest = cache.keys().next().value! + cachedBytes -= cache.get(oldest)!.size + cache.delete(oldest) + } + cache.set(url, blob); cachedBytes += blob.size + return blob + }).finally(() => pending.delete(url)) + pending.set(url, request) + return request +} + +export function hasImageAssets(content: string) { return pattern().test(content) } + +async function replaceAssets(content: string, convert: (blob: Blob) => Promise | string) { + const replacements = new Map() + const references = [...new Map([...content.matchAll(pattern())].map(match => [match[0], match[1]])).entries()] + let cursor = 0 + async function worker() { + while (cursor < references.length) { + const [url, id] = references[cursor++] + replacements.set(url, await convert(await imageBlob(url, id))) + } + } + const results = await Promise.allSettled(Array.from({ length:Math.min(4, references.length) }, worker)) + const failed = results.find(result => result.status === 'rejected') + if (failed?.status === 'rejected') throw failed.reason + return content.replace(pattern(), url => replacements.get(url)!) +} + +export function useImageAssets(source: () => string): { content: Ref; error: Ref; loading: Ref; retry: () => void } { + const content = ref(''), error = ref(''), revision = ref(0), loading = ref(false) + let generation = 0, urls: string[] = [] + watch([source, revision], async ([value]) => { + const current = ++generation, nextUrls: string[] = [] + error.value = ''; loading.value = true + try { + const resolved = await replaceAssets(String(value), blob => { + const url = URL.createObjectURL(blob); nextUrls.push(url); return url + }) + if (current === generation) { + const previous = urls; urls = nextUrls; content.value = resolved + // Give both Vue and markmap time to detach the previous DOM. + window.setTimeout(() => previous.forEach(url => URL.revokeObjectURL(url)), 1000) + } else nextUrls.forEach(url => URL.revokeObjectURL(url)) + } catch { + nextUrls.forEach(url => URL.revokeObjectURL(url)) + if (current === generation) { + content.value = String(value).replace(pattern(), '') + error.value = '图片加载失败,点击重试' + } + } finally { if (current === generation) loading.value = false } + }, { immediate: true }) + onBeforeUnmount(() => { generation++; urls.forEach(url => URL.revokeObjectURL(url)) }) + return { content, error, loading, retry: () => revision.value++ } +} + +export function blobDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(new Error('图片读取失败')) + reader.readAsDataURL(blob) + }) +} + +// Standalone HTML and SVG must not contain browser-session Blob URLs. +export async function embedImages(html: string, svg = false) { + const document = new DOMParser().parseFromString(html, svg ? 'image/svg+xml' : 'text/html') + if (document.querySelector('parsererror')) throw new Error('导出内容格式错误') + for (const image of Array.from(document.querySelectorAll('img'))) { + const src = image.getAttribute('src') || '' + if (src && !src.startsWith('data:')) { + const response = await fetch(src) + if (!response.ok) throw new Error('导出图片加载失败') + image.setAttribute('src', await blobDataUrl(await response.blob())) + } + } + return svg ? new XMLSerializer().serializeToString(document) : document.body.innerHTML +} + +export async function exportMarkdownBundle(markdown: string, filename = 'document.md') { + const result = await hubCall('exports', 'post', { markdown, filename }) + if (!(result.content instanceof ArrayBuffer) || result.mime !== 'application/zip') throw new Error('导出失败') + return new Blob([result.content], { type: result.mime }) +} diff --git a/ui/src/views/mindmap/workbench/utils/markdownAst.ts b/ui/src/views/mindmap/workbench/utils/markdownAst.ts new file mode 100644 index 000000000..ab618ee64 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/markdownAst.ts @@ -0,0 +1,55 @@ +import MarkdownIt from 'markdown-it' +import type Token from 'markdown-it/lib/token.mjs' + +const md = new MarkdownIt({ + html: true, + linkify: true, + typographer: true, + breaks: true +}) + +type AstNode = { + type: string + tag?: string + nesting?: number + level?: number + content?: string + markup?: string + info?: string + block?: boolean + hidden?: boolean + map?: number[] | null + attrs?: Record + children?: AstNode[] +} + +function tokenToAstNode(token: Token): AstNode { + return { + type: token.type, + tag: token.tag || undefined, + nesting: token.nesting, + level: token.level, + content: token.content || undefined, + markup: token.markup || undefined, + info: token.info || undefined, + block: token.block, + hidden: token.hidden, + map: token.map || null, + attrs: token.attrs ? Object.fromEntries(token.attrs) : undefined, + children: token.children?.map(tokenToAstNode) + } +} + +export function markdownToAstString(markdown: string): string { + if (!markdown.trim()) { + return '' + } + + const tokens = md.parse(markdown, {}) + const ast = { + type: 'root', + children: tokens.map(tokenToAstNode) + } + + return JSON.stringify(ast, null, 2) +} diff --git a/ui/src/views/mindmap/workbench/utils/markdownTemplate.ts b/ui/src/views/mindmap/workbench/utils/markdownTemplate.ts new file mode 100644 index 000000000..1154e82f6 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/markdownTemplate.ts @@ -0,0 +1,45 @@ +export interface ReplacementRule { + id: string + search: string + replace: string +} + +export interface MarkdownTemplateRenderResult { + markdown: string + error: string | null +} + +export function createEmptyReplacementRule(): ReplacementRule { + return { + id: `${Date.now()}_${Math.random().toString(16).slice(2)}`, + search: '', + replace: '' + } +} + +export function renderMarkdownTemplate(template: string, rules: ReplacementRule[]): MarkdownTemplateRenderResult { + if (!template.trim()) { + return { + markdown: '', + error: null + } + } + + const activeRules = rules.filter((rule) => rule.search.trim()) + + try { + const markdown = activeRules.reduce((current, rule) => { + return current.split(rule.search).join(rule.replace) + }, template) + + return { + markdown, + error: null + } + } catch (error) { + return { + markdown: template, + error: `替换规则执行失败:${(error as Error).message}` + } + } +} diff --git a/ui/src/views/mindmap/workbench/utils/mindmapMarkdown.ts b/ui/src/views/mindmap/workbench/utils/mindmapMarkdown.ts new file mode 100644 index 000000000..ab77f4cc1 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/mindmapMarkdown.ts @@ -0,0 +1,175 @@ +import MarkdownIt from 'markdown-it' +import type Token from 'markdown-it/lib/token.mjs' + +const ROOT_TITLE = '文档思维导图' +const MAX_HEADING_LEVEL = 6 + +const md = new MarkdownIt({ + html: true, + linkify: true, + typographer: true, + breaks: true +}) + +function normalizeNodeText(text: string) { + return text + .replace(/\r?\n/g, ' ') + .replace(/\s+/g, ' ') + .trim() +} + +function clampHeadingLevel(level: number) { + return Math.min(Math.max(level, 1), MAX_HEADING_LEVEL) +} + +function pushHeading(result: string[], level: number, text: string) { + const normalized = normalizeNodeText(text) + if (!normalized) return + result.push(`${'#'.repeat(clampHeadingLevel(level))} ${normalized}`) +} + +function getHeadingLevel(token: Token) { + const tagLevel = Number(token.tag.replace(/^h/i, '')) + return Number.isFinite(tagLevel) && tagLevel > 0 ? tagLevel : 1 +} + +function getNextInlineContent(tokens: Token[], index: number) { + const next = tokens[index + 1] + return next?.type === 'inline' ? next.content : '' +} + +function getFenceContent(token: Token) { + const info = normalizeNodeText(token.info || '') + const content = token.content.trim() + if (!content) return info ? `代码块 ${info}` : '代码块' + return info ? `代码块 ${info}: ${content}` : `代码块: ${content}` +} + +function collectTableRows(tokens: Token[], startIndex: number) { + const rows: string[][] = [] + let currentRow: string[] | null = null + let endIndex = startIndex + + for (let i = startIndex + 1; i < tokens.length; i += 1) { + const token = tokens[i] + endIndex = i + + if (token.type === 'table_close') break + if (token.type === 'tr_open') { + currentRow = [] + continue + } + if (token.type === 'tr_close') { + if (currentRow && currentRow.some((cell) => cell.trim())) { + rows.push(currentRow) + } + currentRow = null + continue + } + if ((token.type === 'th_open' || token.type === 'td_open') && currentRow) { + currentRow.push(getNextInlineContent(tokens, i)) + } + } + + return { rows, endIndex } +} + +export function documentTitleFromName(name: string) { + return name.replace(/\.[^.\\/]+$/, '').trim() || ROOT_TITLE +} + +export function buildMindmapMarkdown(markdown: string, title = ROOT_TITLE) { + if (!markdown.trim()) return '' + + const tokens = md.parse(markdown, {}) + const rootTitle = normalizeNodeText(title).replace(/([\\`*{}\[\]()#+.!_<>])/g, '\\$1') + const result: string[] = [`# ${rootTitle || ROOT_TITLE}`] + let currentHeadingLevel = 1 + let listDepth = 0 + + for (let i = 0; i < tokens.length; i += 1) { + const token = tokens[i] + + if (token.type === 'heading_open') { + currentHeadingLevel = clampHeadingLevel(getHeadingLevel(token) + 1) + pushHeading(result, currentHeadingLevel, getNextInlineContent(tokens, i) || '未命名章节') + i += 2 + continue + } + + if (token.type === 'bullet_list_open' || token.type === 'ordered_list_open') { + listDepth += 1 + continue + } + + if (token.type === 'bullet_list_close' || token.type === 'ordered_list_close') { + listDepth = Math.max(0, listDepth - 1) + continue + } + + if (token.type === 'paragraph_open') { + const level = currentHeadingLevel + Math.max(listDepth, 1) + pushHeading(result, level, getNextInlineContent(tokens, i)) + i += 2 + continue + } + + if (token.type === 'fence' || token.type === 'code_block') { + pushHeading(result, currentHeadingLevel + 1, getFenceContent(token)) + continue + } + + if (token.type === 'html_block') { + pushHeading(result, currentHeadingLevel + 1, token.content) + continue + } + + if (token.type === 'blockquote_open') { + pushHeading(result, currentHeadingLevel + 1, '引用') + currentHeadingLevel = clampHeadingLevel(currentHeadingLevel + 1) + continue + } + + if (token.type === 'blockquote_close') { + currentHeadingLevel = Math.max(1, currentHeadingLevel - 1) + continue + } + + if (token.type === 'hr') { + pushHeading(result, currentHeadingLevel + 1, '---') + continue + } + + if (token.type === 'table_open') { + const tableLevel = currentHeadingLevel + 1 + const { rows, endIndex } = collectTableRows(tokens, i) + pushHeading(result, tableLevel, '表格') + rows.forEach((row) => { + pushHeading(result, tableLevel + 1, row.map(normalizeNodeText).join(' | ')) + }) + i = endIndex + } + } + + return result.join('\n') +} + +export function replaceFirstMindmapText(markdown: string, oldText: string, newText: string) { + const normalizedOld = oldText.replace(/^(引用|表格|代码块)(\s|:|:)?/, '').trim() + if (!normalizedOld || !newText.trim()) return markdown + + const escaped = normalizedOld.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const directMatch = new RegExp(escaped) + if (directMatch.test(markdown)) { + return markdown.replace(directMatch, newText.trim()) + } + + const lines = markdown.split(/\r?\n/) + const targetLineIndex = lines.findIndex((line) => normalizeNodeText(line).includes(normalizedOld)) + if (targetLineIndex >= 0) { + lines[targetLineIndex] = lines[targetLineIndex].replace(normalizedOld, newText.trim()) + return lines.join('\n') + } + + return markdown +} diff --git a/ui/src/views/mindmap/workbench/utils/mindmapState.ts b/ui/src/views/mindmap/workbench/utils/mindmapState.ts new file mode 100644 index 000000000..ea07243d5 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/mindmapState.ts @@ -0,0 +1,46 @@ +export interface MindmapNode { + content?: string + children?: MindmapNode[] + payload?: { fold?: number; [key: string]: unknown } +} + +export interface FoldState { paths: Map; unique: Map } + +function identity(node: MindmapNode) { + // Browser Blob URLs change on every preview render; image identity must not. + return (node.content || '').replace(/src="blob:[^"]*"/g, 'src=""').replace(/\s+/g, ' ').trim() +} + +function visit(root: MindmapNode, fn: (node: MindmapNode, path: string, text: string) => void) { + const walk = (node: MindmapNode, path: string) => { + fn(node, path, identity(node)) + const occurrences = new Map() + for (const child of node.children || []) { + const text = identity(child), ordinal = occurrences.get(text) || 0 + occurrences.set(text, ordinal + 1) + walk(child, `${path}/${JSON.stringify([text, ordinal])}`) + } + } + walk(root, 'root') +} + +export function captureFolds(root: MindmapNode): FoldState { + const paths = new Map(), unique = new Map(), repeated = new Set() + visit(root, (node, path, text) => { + const fold = node.payload?.fold || 0 + paths.set(path, fold) + if (unique.has(text)) repeated.add(text) + else unique.set(text, fold) + }) + repeated.forEach(text => unique.delete(text)) + return { paths, unique } +} + +export function restoreFolds(root: MindmapNode, saved?: FoldState) { + const counts = new Map() + visit(root, (_node, _path, text) => counts.set(text, (counts.get(text) || 0) + 1)) + visit(root, (node, path, text) => { + const fold = saved?.paths.get(path) ?? (counts.get(text) === 1 ? saved?.unique.get(text) : undefined) ?? 0 + node.payload = { ...node.payload, fold: node.children?.length ? fold : 0 } + }) +} diff --git a/ui/src/views/mindmap/workbench/utils/pdfExport.ts b/ui/src/views/mindmap/workbench/utils/pdfExport.ts new file mode 100644 index 000000000..d5c21d387 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/pdfExport.ts @@ -0,0 +1,16 @@ +export async function exportElementPdf(element: HTMLElement): Promise { + const [{ default: html2canvas }, { jsPDF }] = await Promise.all([import('html2canvas'), import('jspdf')]) + const canvas = await html2canvas(element, { scale: 2, useCORS: true, backgroundColor: '#ffffff' }) + const pdf = new jsPDF('p', 'mm', 'a4') + const width = pdf.internal.pageSize.getWidth() - 20 + const pageHeight = pdf.internal.pageSize.getHeight() - 20 + const sliceHeight = Math.floor(canvas.width * pageHeight / width) + for (let y = 0; y < canvas.height; y += sliceHeight) { + if (y) pdf.addPage() + const slice = document.createElement('canvas') + slice.width = canvas.width; slice.height = Math.min(sliceHeight, canvas.height - y) + slice.getContext('2d')!.drawImage(canvas, 0, y, canvas.width, slice.height, 0, 0, canvas.width, slice.height) + pdf.addImage(slice.toDataURL('image/jpeg', .95), 'JPEG', 10, 10, width, slice.height * width / canvas.width) + } + return pdf.output('blob') +} diff --git a/ui/src/views/mindmap/workbench/utils/sanitize.ts b/ui/src/views/mindmap/workbench/utils/sanitize.ts new file mode 100644 index 000000000..96995b411 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/sanitize.ts @@ -0,0 +1,14 @@ +import DOMPurify from 'dompurify' + +export function sanitizeHtml(html: string): string { + return DOMPurify.sanitize(html, { + USE_PROFILES: { html: true, svg: true, mathMl: true }, + FORBID_TAGS: ['style', 'form', 'input', 'button', 'iframe'], + ALLOWED_URI_REGEXP: /^(?:(?:https?|mailto|tel|blob):|data:image\/(?:png|jpeg|gif|webp|bmp);base64,|[^a-z]|[a-z+.-]+(?:[^a-z+.-:]|$))/i + }) +} + +export function sanitizeMindmap(node: { content: string; children?: any[] }) { + node.content = sanitizeHtml(node.content) + node.children?.forEach(sanitizeMindmap) +} diff --git a/ui/src/views/mindmap/workbench/utils/uuid.ts b/ui/src/views/mindmap/workbench/utils/uuid.ts new file mode 100644 index 000000000..9f0f82732 --- /dev/null +++ b/ui/src/views/mindmap/workbench/utils/uuid.ts @@ -0,0 +1,10 @@ +// crypto.randomUUID 仅在安全上下文(HTTPS/localhost)可用;管理平台可能以 HTTP 访问,需要降级。 +export function uuid(): string { + if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') return crypto.randomUUID() + const buf = new Uint8Array(16) + crypto.getRandomValues(buf) + buf[6] = (buf[6] & 0x0f) | 0x40 + buf[8] = (buf[8] & 0x3f) | 0x80 + const hex = Array.from(buf, b => b.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} diff --git a/ui/src/views/mindmap/workbench/views/DocumentProcessor.vue b/ui/src/views/mindmap/workbench/views/DocumentProcessor.vue new file mode 100644 index 000000000..b5daf7649 --- /dev/null +++ b/ui/src/views/mindmap/workbench/views/DocumentProcessor.vue @@ -0,0 +1,1370 @@ + + + + + diff --git a/ui/src/views/system-setting/policy/index.vue b/ui/src/views/system-setting/policy/index.vue index 6342fcd46..a261f63fc 100644 --- a/ui/src/views/system-setting/policy/index.vue +++ b/ui/src/views/system-setting/policy/index.vue @@ -33,14 +33,28 @@ - - + + - +