设置
+暂无转换结果
+diff --git a/apps/application/sql/list_application_user.sql b/apps/application/sql/list_application_user.sql index a1a37e3fa..9923ca0d7 100644 --- a/apps/application/sql/list_application_user.sql +++ b/apps/application/sql/list_application_user.sql @@ -16,6 +16,6 @@ from (select application."id"::text, application."name", from application left join "user" on user_id = "user".id where application."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list))) temp ${application_query_set} diff --git a/apps/application/sql/list_application_user_ee.sql b/apps/application/sql/list_application_user_ee.sql index bfab07b33..6601376a6 100644 --- a/apps/application/sql/list_application_user_ee.sql +++ b/apps/application/sql/list_application_user_ee.sql @@ -16,7 +16,7 @@ from (select application."id"::text, application."name", from application left join "user" on user_id = "user".id where "application".id::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then 'ROLE' = any (permission_list) @@ -28,9 +28,9 @@ from (select application."id"::text, application."name", from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id = role_permission.role_id - where user_role_relation.user_id = workspace_user_resource_permission.user_id + where user_role_relation.user_id = effective_resource_permission.user_id and user_role_relation.workspace_id = - workspace_user_resource_permission.workspace_id) + effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) diff --git a/apps/application/views/application.py b/apps/application/views/application.py index 518b74c34..cee7357e8 100644 --- a/apps/application/views/application.py +++ b/apps/application/views/application.py @@ -272,6 +272,8 @@ class McpServers(APIView): CompareConstants.AND), RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) def post(self, request: Request, workspace_id, application_id: str): + from system_manage.access_control import enforce + enforce(request, 'application', application_id) return result.success(ApplicationOperateSerializer( data={'mcp_servers': request.query_params.get('mcp_servers'), 'workspace_id': workspace_id, 'user_id': request.user.id, diff --git a/apps/application/views/application_chat.py b/apps/application/views/application_chat.py index d60206ded..66c85d827 100644 --- a/apps/application/views/application_chat.py +++ b/apps/application/views/application_chat.py @@ -132,6 +132,8 @@ class OpenView(APIView): CompareConstants.AND), RoleConstants.WORKSPACE_MANAGE.get_workspace_role()) def get(self, request: Request, workspace_id: str, application_id: str): + from system_manage.access_control import enforce + enforce(request, 'application', application_id) return result.success(OpenChatSerializers( data={'workspace_id': workspace_id, 'application_id': application_id, 'chat_user_id': str(uuid.uuid7()), 'chat_user_type': ChatUserType.ANONYMOUS_USER, @@ -152,7 +154,7 @@ class ChatView(APIView): tags=[_('Application')] # type: ignore ) def post(self, request: Request, chat_id: str): - return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data) + return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data, request=request) class PromptGenerateView(APIView): authentication_classes = [TokenAuth] @@ -176,4 +178,4 @@ class PromptGenerateView(APIView): @log(menu='Application', operate='Generate prompt', get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id'))) def post(self, request: Request, workspace_id: str, model_id:str, application_id: str): - return PromptGenerateSerializer(data={'workspace_id': workspace_id, 'model_id': model_id, 'application_id': application_id}).generate_prompt(instance=request.data) \ No newline at end of file + return PromptGenerateSerializer(data={'workspace_id': workspace_id, 'model_id': model_id, 'application_id': application_id}).generate_prompt(instance=request.data) diff --git a/apps/chat/serializers/chat.py b/apps/chat/serializers/chat.py index 28e361292..c05a517dc 100644 --- a/apps/chat/serializers/chat.py +++ b/apps/chat/serializers/chat.py @@ -131,10 +131,12 @@ def get_post_handler(chat_info: ChatInfo): class DebugChatSerializers(serializers.Serializer): chat_id = serializers.UUIDField(required=True, label=_("Conversation ID")) - def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse()): + def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse(), request=None): self.is_valid(raise_exception=True) chat_id = self.data.get('chat_id') chat_info: ChatInfo = ChatInfo.get_cache(chat_id) + from system_manage.access_control import enforce + enforce(request, 'application', chat_info.application_id) application = QuerySet(Application).filter(id=chat_info.application_id).first() chat_info.application = application return ChatSerializers(data={ diff --git a/apps/chat/serializers/chat_authentication.py b/apps/chat/serializers/chat_authentication.py index 7d9c18e74..decba655b 100644 --- a/apps/chat/serializers/chat_authentication.py +++ b/apps/chat/serializers/chat_authentication.py @@ -86,6 +86,8 @@ class AnonymousAuthenticationSerializer(serializers.Serializer): access_token = self.data.get("access_token") application_access_token = QuerySet(ApplicationAccessToken).filter(access_token=access_token).first() if application_access_token is not None and application_access_token.is_active: + from system_manage.access_control import enforce + enforce(request, 'application', application_access_token.application_id) api_key = self.data.get('api_key') or self.get_request_api_key(request) if api_key: application_api_key = self.validate_application_api_key(application_access_token.application_id, api_key) @@ -121,6 +123,8 @@ class AuthProfileSerializer(serializers.Serializer): if not application_access_token.is_active: raise NotFound404(404, _("Invalid access_token")) application_id = application_access_token.application_id + from system_manage.access_control import enforce + enforce(request, 'application', application_id) api_key = self.data.get("api_key") or AnonymousAuthenticationSerializer.get_request_api_key(request) if api_key: AnonymousAuthenticationSerializer.validate_application_api_key(application_id, api_key) diff --git a/apps/common/auth/authenticate.py b/apps/common/auth/authenticate.py index a762fed11..a282f7aa6 100644 --- a/apps/common/auth/authenticate.py +++ b/apps/common/auth/authenticate.py @@ -85,7 +85,17 @@ class TokenAuth(TokenAuthentication): token_details = TokenDetails(token) for handle in handles: if handle.support(request, token, token_details.get_token_details): - return handle.handle(request, token, token_details.get_token_details) + authenticated = handle.handle(request, token, token_details.get_token_details) + from common.constants.permission_constants import ChatAuth + if isinstance(authenticated[1], ChatAuth): + from system_manage.access_control import enforce + match = getattr(request, 'resolver_match', None) + routed_id = match.kwargs.get('application_id') if match else None + if routed_id and str(routed_id) != str(authenticated[1].application_id): + from common.exception.app_exception import AppUnauthorizedFailed + raise AppUnauthorizedFailed(403, '无权访问该助手') + enforce(request, 'application', authenticated[1].application_id) + return authenticated raise AppAuthenticationFailed(1002, _('Authentication information is incorrect! illegal user')) except Exception as e: traceback.print_stack() diff --git a/apps/common/auth/handle/impl/user_token.py b/apps/common/auth/handle/impl/user_token.py index f34136547..4e066dca1 100644 --- a/apps/common/auth/handle/impl/user_token.py +++ b/apps/common/auth/handle/impl/user_token.py @@ -275,6 +275,10 @@ def get_role_list(user, def get_auth(user): + from system_manage.platform_access import is_admin + # The old field is a compatibility projection, never an authority. + administrator = is_admin(user) + user.role = 'ADMIN' if administrator else 'USER' workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping") workspace_model = DatabaseModelManage.get_model("workspace_model") role_model = DatabaseModelManage.get_model("role_model") @@ -284,7 +288,15 @@ def get_auth(user): role_model, role_permission_mapping_model) role_list = get_role_list(user, workspace_user_role_mapping_model, workspace_model, role_model, role_permission_mapping_model) - return Auth(role_list, permission_list) + if workspace_user_role_mapping_model is None: + role_list = ['ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')] if administrator else ['USER', get_role_permission(RoleConstants.USER, 'default')] + elif administrator: + role_list = list(set(role_list) | {'ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')}) + else: + role_list = [role for role in role_list if role not in ('ADMIN', 'EXTENDS_ADMIN')] + from system_manage.platform_access import permission_strings + # Evaluate new grants live, independently of legacy cached permissions. + return Auth(role_list, list(set(permission_list) | permission_strings(user))) class UserToken(AuthBaseHandle): @@ -306,4 +318,6 @@ class UserToken(AuthBaseHandle): if not user.is_active or user.password != cache_token.password: raise AppAuthenticationFailed(1002, _('Authentication information is incorrect')) auth = get_auth(user) + from system_manage.platform_access import check_feature_request + check_feature_request(request, user) return user, auth diff --git a/apps/folders/serializers/folder.py b/apps/folders/serializers/folder.py index 790a913ca..bca0772a2 100644 --- a/apps/folders/serializers/folder.py +++ b/apps/folders/serializers/folder.py @@ -16,7 +16,7 @@ from folders.api.folder import FolderCreateRequest from knowledge.models import KnowledgeFolder, Knowledge from knowledge.serializers.knowledge import KnowledgeSerializer from knowledge.serializers.knowledge_folder import KnowledgeFolderTreeSerializer -from system_manage.models import WorkspaceUserResourcePermission +from system_manage.models import WorkspaceUserResourcePermission, EffectiveResourcePermission from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer from tools.models import ToolFolder, Tool from tools.serializers.tool import ToolSerializer @@ -101,7 +101,7 @@ def get_max_depth(current_node): def has_target_permission(workspace_id, source, user_id, target): - return QuerySet(WorkspaceUserResourcePermission).filter(workspace_id=workspace_id, user_id=user_id, + return QuerySet(EffectiveResourcePermission).filter(workspace_id=workspace_id, user_id=user_id, auth_target_type=source, target=target, permission_list__contains=['MANAGE']).exists() @@ -249,13 +249,13 @@ class FolderSerializer(serializers.Serializer): .annotate(id_str=Cast('id', TextField())) .values_list('id_str', flat=True)) # 检查文件夹是否存在未授权当前用户的资源 - auth_list = QuerySet(WorkspaceUserResourcePermission).filter( + auth_list = QuerySet(EffectiveResourcePermission).filter( Q(workspace_id=self.data.get('workspace_id')) & Q(user_id=self.data.get('user_id')) & Q(auth_target_type=self.data.get('source')) & Q(target__in=source_ids) & Q(permission_list__overlap=[ResourcePermission.MANAGE, ResourcePermissionRole.ROLE]) - ).count() + ).values('target').distinct().count() if auth_list != len(source_ids): raise AppApiException(500, _('This folder contains resources that you dont have permission')) self.delete_source(node) @@ -319,7 +319,7 @@ class FolderTreeSerializer(serializers.Serializer): if name is not None: base_q &= Q(name__contains=name) if not workspace_manage: - base_q &= (Q(id__in=WorkspaceUserResourcePermission.objects.filter(user_id=current_user.id, + base_q &= (Q(id__in=EffectiveResourcePermission.objects.filter(user_id=current_user.id, auth_target_type=self.data.get('source'), workspace_id=self.data.get( 'workspace_id'), diff --git a/apps/knowledge/serializers/document.py b/apps/knowledge/serializers/document.py index 0df156081..528920349 100644 --- a/apps/knowledge/serializers/document.py +++ b/apps/knowledge/serializers/document.py @@ -1066,6 +1066,13 @@ class DocumentSerializers(serializers.Serializer): raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度") file_list = instance.get("file") + from system_manage.access_control import enforce + from system_manage.parsers import extension, MINERU_INPUTS + request = self.context.get('request') + enforce(request, 'knowledge', self.data.get('knowledge_id')) + parser_id = (knowledge.meta or {}).get('parser_id') if knowledge else None + if parser_id and any(extension(file.name) in MINERU_INPUTS for file in file_list): + enforce(request, 'parser', parser_id) return reduce( lambda x, y: [*x, *y], [self.file_to_paragraph( diff --git a/apps/knowledge/sql/list_knowledge_user.sql b/apps/knowledge/sql/list_knowledge_user.sql index 55c638a97..82b019c76 100644 --- a/apps/knowledge/sql/list_knowledge_user.sql +++ b/apps/knowledge/sql/list_knowledge_user.sql @@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, FROM (SELECT knowledge.* FROM knowledge knowledge ${knowledge_custom_sql} AND id::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list))) temp_knowledge LEFT JOIN (SELECT "count"("id") AS document_count, "sum"("char_length") "char_length", knowledge_id FROM "document" diff --git a/apps/knowledge/sql/list_knowledge_user_ee.sql b/apps/knowledge/sql/list_knowledge_user_ee.sql index 7f76d8a5e..2e22d7b24 100644 --- a/apps/knowledge/sql/list_knowledge_user_ee.sql +++ b/apps/knowledge/sql/list_knowledge_user_ee.sql @@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, FROM (SELECT knowledge.* FROM knowledge knowledge ${knowledge_custom_sql} AND "knowledge".id::text in (select target - from workspace_user_resource_permission + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then @@ -34,8 +34,8 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name, from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id=role_permission.role_id - where user_role_relation.user_id=workspace_user_resource_permission.user_id - and user_role_relation.workspace_id=workspace_user_resource_permission.workspace_id) + where user_role_relation.user_id=effective_resource_permission.user_id + and user_role_relation.workspace_id=effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) end diff --git a/apps/knowledge/views/document.py b/apps/knowledge/views/document.py index b37800899..f4edf1255 100644 --- a/apps/knowledge/views/document.py +++ b/apps/knowledge/views/document.py @@ -194,7 +194,7 @@ class DocumentView(APIView): return result.success(DocumentSerializers.Split(data={ 'workspace_id': workspace_id, 'knowledge_id': knowledge_id, - }, context={'user': request.user}).parse(split_data)) + }, context={'user': request.user, 'request': request}).parse(split_data)) class SplitPattern(APIView): authentication_classes = [TokenAuth] diff --git a/apps/models_provider/sql/list_model_user.sql b/apps/models_provider/sql/list_model_user.sql index 15dbf21d7..f9a8faa68 100644 --- a/apps/models_provider/sql/list_model_user.sql +++ b/apps/models_provider/sql/list_model_user.sql @@ -15,5 +15,5 @@ FROM (SELECT model."id"::text, model."name", from model left join "user" on user_id = "user".id where model."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and 'VIEW' = any (permission_list)) ) temp ${model_query_set} diff --git a/apps/models_provider/sql/list_model_user_ee.sql b/apps/models_provider/sql/list_model_user_ee.sql index 5febcc418..5b82da1d9 100644 --- a/apps/models_provider/sql/list_model_user_ee.sql +++ b/apps/models_provider/sql/list_model_user_ee.sql @@ -15,7 +15,7 @@ FROM (SELECT model."id"::text, model."name", from model left join "user" on user_id = "user".id where model."id"::text in (select target - from workspace_user_resource_permission ${workspace_user_resource_permission_query_set} + from effective_resource_permission ${workspace_user_resource_permission_query_set} and case when auth_type = 'ROLE' then 'ROLE' = any (permission_list) @@ -27,9 +27,9 @@ FROM (SELECT model."id"::text, model."name", from role_permission role_permission right join user_role_relation user_role_relation on user_role_relation.role_id = role_permission.role_id - where user_role_relation.user_id = workspace_user_resource_permission.user_id + where user_role_relation.user_id = effective_resource_permission.user_id and user_role_relation.workspace_id = - workspace_user_resource_permission.workspace_id) + effective_resource_permission.workspace_id) else 'VIEW' = any (permission_list) diff --git a/apps/system_manage/access_control.py b/apps/system_manage/access_control.py new file mode 100644 index 000000000..9effeb1be --- /dev/null +++ b/apps/system_manage/access_control.py @@ -0,0 +1,235 @@ +"""Request-side access policy enforcement, using existing settings and audit tables.""" +import ipaddress +import os +from datetime import timedelta +from zoneinfo import ZoneInfo + +from django.db import transaction +from django.utils import timezone +from rest_framework.exceptions import ValidationError +from common.exception.app_exception import AppUnauthorizedFailed +from system_manage.models import AccessControlPolicy, AccessControlPolicyApplicationRecord, SystemSetting +from system_manage.models.log_management import Log + +# A separate settings row; no schema change or automatic migration is needed. +SETTING_ID = 20 +POLICY_TIMEZONE = ZoneInfo('Asia/Shanghai') +DEVICE_TYPES = ('桌面端', '移动终端', '平板', '服务器') + + +def configuration(lock=False): + query = SystemSetting.objects + if lock: + query.get_or_create(type=SETTING_ID, defaults={'meta': {'bindings': [], 'region_networks': []}}) + query = query.select_for_update() + row = query.filter(type=SETTING_ID).first() + return row, dict(row.meta or {}) if row else {'bindings': [], 'region_networks': []} + + +def network(value): + try: + return str(ipaddress.ip_network(value.strip(), strict=False)) + except (ValueError, AttributeError): + raise ValidationError('请输入有效的 IP 或网段') + + +def client_ip(request): + """Only consume forwarded addresses from explicitly trusted proxy chains.""" + try: + peer = ipaddress.ip_address(request.META.get('REMOTE_ADDR', '')) + except ValueError: + return '' + trusted = [] + for value in os.environ.get('MAXKB_ACCESS_TRUSTED_PROXIES', '').split(','): + if value.strip(): + trusted.append(ipaddress.ip_network(value.strip(), strict=False)) + forwarded = request.META.get('HTTP_X_FORWARDED_FOR', '') + if any(peer in block for block in trusted) and forwarded: + for value in reversed(forwarded.split(',')): + if not any(peer in block for block in trusted): + break + try: + peer = ipaddress.ip_address(value.strip()) + except ValueError: + raise AppUnauthorizedFailed(403, '接入受限:来源 IP 无效') + return str(peer) + + +def device_type(request): + agent = request.META.get('HTTP_USER_AGENT', '').lower() + if any(token in agent for token in ('ipad', 'tablet')) or ('android' in agent and 'mobile' not in agent): + return '平板' + if any(token in agent for token in ('mobile', 'iphone', 'android')): + return '移动终端' + if any(token in agent for token in ('mozilla', 'windows', 'macintosh', 'x11')): + return '桌面端' + if any(token in agent for token in ('curl', 'python', 'java/', 'go-http', 'postman', 'wget', 'node')): + return '服务器' + return '未知' + + +def contains(address, block): + try: + return ipaddress.ip_address(address) in ipaddress.ip_network(block, strict=False) + except ValueError: + return False + + +def policy_reason(policy, now, device, regions): + """Empty reason means allowed. Disabled / out-of-effect policies do not restrict.""" + if not policy.enabled or (policy.effect_start_time and now < policy.effect_start_time) or ( + policy.effect_end_time and now >= policy.effect_end_time): + return '' + local = now.astimezone(POLICY_TIMEZONE) + if policy.access_time_enabled: + start, end = policy.access_start_time, policy.access_end_time + if start is None or end is None or start == end: + return '访问时段配置无效' + current = local.time().replace(tzinfo=None) + if start < end: + in_period = start <= current < end + day = local.isoweekday() + else: + in_period = current >= start or current < end + day = (local - timedelta(days=1)).isoweekday() if current < end else local.isoweekday() + if not in_period or (policy.allowed_weekdays and day not in policy.allowed_weekdays): + return '不在允许访问时段' + if policy.allowed_device_types and device not in policy.allowed_device_types: + return '设备类型不允许' + if policy.allowed_regions and not set(policy.allowed_regions).intersection(regions): + return '来源区域不允许或无法识别' + return '' + + +def enforce(request, kind, identifier): + if request is None: + return + target = f'{kind}:{identifier}' + checked = getattr(request, '_access_policy_checked', set()) + if target in checked: + return + _, config = configuration() + if not config.get('bindings'): + return + try: + address = client_ip(request) + except AppUnauthorizedFailed: + Log.objects.create(menu='接入控制', operate='拒绝访问', status=403, + operation_object={'target': target}, user={}, ip_address=request.META.get('REMOTE_ADDR', '')[:128], + details={'path': request.path, 'reason': '来源 IP 无效', 'device_type': device_type(request)}) + raise + bindings = [b for b in config.get('bindings', []) if ( + b['target_type'] == 'SERVICE' and b['target_identifier'] == target) or ( + b['target_type'] == 'DEVICE' and contains(address, b['target_identifier']))] + if not bindings: + return + device = device_type(request) + regions = {item['name'] for item in config.get('region_networks', []) if contains(address, item['network'])} + policies = list(AccessControlPolicy.objects.filter(id__in={b['policy_id'] for b in bindings}).order_by('id')) + reason, denied_policy = '', None + now = timezone.now() + for policy in policies: + reason = policy_reason(policy, now, device, regions) + if reason: + denied_policy = policy + break + # DRF has not assigned request.user while an authentication handler runs. + user = request.__dict__.get('_user') + Log.objects.create(menu='接入控制', operate='拒绝访问' if reason else '允许访问', + operation_object={'target': target, 'policy_id': str(denied_policy.id) if denied_policy else ''}, + user={'id': str(user.id)} if getattr(user, 'id', None) else {}, status=403 if reason else 200, + ip_address=address, workspace_id='default', + details={'path': request.path, 'device_type': device, 'regions': sorted(regions), + 'policies': [p.name for p in policies], 'reason': reason}) + if reason: + raise AppUnauthorizedFailed(403, f'接入受限:{reason}') + request._access_policy_checked = checked | {target} + + +def targets(): + from application.models import Application + from system_manage.models.ai_hub import AIService, AIParserResource + return [{'id': f'{kind}:{obj.id}', 'name': obj.name, 'kind': kind, + 'label': f'{label} · {obj.name}'} + for kind, label, model in (('application', '问答助手', Application), + ('mindmap', '导图服务', AIService), ('parser', '解析服务', AIParserResource)) + for obj in model.objects.all().order_by('name')] + + +def enforce_task(request, service_id=None, nodes=None): + """Call before task transactions so denied-call audit survives their rollback.""" + enforce(request, 'mindmap', service_id or 'workbench') + for node in nodes if isinstance(nodes, list) else []: + if isinstance(node, dict) and node.get('type') == 'parse' and node.get('resource_id'): + enforce(request, 'parser', node['resource_id']) + + +@transaction.atomic +def bind(policy_id, operator, data): + policy = AccessControlPolicy.objects.select_for_update().get(pk=policy_id) + kind = data['target_type'] + target = data['target_identifier'].strip() + if kind == 'DEVICE': + target = network(target) + name = data.get('target_name', '').strip() or target + else: + obj = next((item for item in targets() if item['id'] == target), None) + if obj is None: + raise ValidationError('请选择有效服务') + name = obj['label'][:128] + row, config = configuration(lock=True) + item = {'policy_id': str(policy.id), 'target_type': kind, 'target_identifier': target, + 'target_name': name, 'operator_name': operator.nick_name or operator.username, + 'apply_time': timezone.now().isoformat()} + bindings = config.setdefault('bindings', []) + if any(all(b[key] == item[key] for key in ('policy_id', 'target_type', 'target_identifier')) for b in bindings): + raise ValidationError('该对象已绑定此策略') + bindings.append(item) + row.meta = config + row.save(update_fields=['meta', 'update_time']) + AccessControlPolicyApplicationRecord.objects.create(policy=policy, policy_name=policy.name, + target_type=kind, target_identifier=target, target_name=name, + operator=operator, operator_name=item['operator_name'], remark='绑定策略;' + data.get('remark', '')) + policy.application_count += 1 + policy.last_applied_time = timezone.now() + policy.save(update_fields=['application_count', 'last_applied_time', 'update_time']) + return item + + +@transaction.atomic +def unbind(policy_id, operator, data): + policy = AccessControlPolicy.objects.select_for_update().get(pk=policy_id) + row, config = configuration(lock=True) + items = config.get('bindings', []) + removed = [b for b in items if b['policy_id'] == str(policy.id) + and b['target_type'] == data.get('target_type') and b['target_identifier'] == data.get('target_identifier')] + if not removed: + raise ValidationError('绑定不存在') + config['bindings'] = [b for b in items if b not in removed] + row.meta = config + row.save(update_fields=['meta', 'update_time']) + item = removed[0] + AccessControlPolicyApplicationRecord.objects.create(policy=policy, policy_name=policy.name, + target_type=item['target_type'], target_identifier=item['target_identifier'], target_name=item['target_name'], + operator=operator, operator_name=operator.nick_name or operator.username, remark='解除绑定') + + +@transaction.atomic +def save_regions(data): + if not isinstance(data, list) or len(data) > 500: + raise ValidationError('区域网段最多500项') + clean = [] + for item in data: + if not isinstance(item, dict) or not isinstance(item.get('name'), str) or not 1 <= len(item['name'].strip()) <= 64: + raise ValidationError('请输入有效区域名称') + block = network(item.get('network')) + for prior in clean: + left, right = ipaddress.ip_network(prior['network']), ipaddress.ip_network(block) + if left.version == right.version and left.overlaps(right): + raise ValidationError('区域网段不能重叠') + clean.append({'name': item['name'].strip(), 'network': block}) + row, config = configuration(lock=True) + config['region_networks'] = clean + row.meta = config + row.save(update_fields=['meta', 'update_time']) + return clean diff --git a/apps/system_manage/access_control_decisions.py b/apps/system_manage/access_control_decisions.py new file mode 100644 index 000000000..fefdda662 --- /dev/null +++ b/apps/system_manage/access_control_decisions.py @@ -0,0 +1,49 @@ +"""Validated, paginated access decision queries (no configuration mutations).""" +from zoneinfo import ZoneInfo + +from rest_framework import serializers + + +class DecisionQuery(serializers.Serializer): + page = serializers.IntegerField(min_value=1, default=1) + page_size = serializers.IntegerField(min_value=1, max_value=100, default=20) + start_time = serializers.DateTimeField(required=False, default_timezone=ZoneInfo('Asia/Shanghai')) + end_time = serializers.DateTimeField(required=False, default_timezone=ZoneInfo('Asia/Shanghai')) + result = serializers.ChoiceField(choices=['allowed', 'denied'], required=False, allow_blank=True) + ip = serializers.CharField(required=False, allow_blank=True, max_length=128) + device_type = serializers.ChoiceField( + choices=['桌面端', '移动终端', '平板', '服务器', '未知'], required=False, allow_blank=True) + reason = serializers.CharField(required=False, allow_blank=True, max_length=200) + denied = serializers.BooleanField(required=False) + + def validate(self, data): + if data.get('start_time') and data.get('end_time') and data['start_time'] > data['end_time']: + raise serializers.ValidationError({'end_time': '结束时间不能早于开始时间'}) + return data + + +def list_decisions(params): + from system_manage.models.log_management import Log + + serializer = DecisionQuery(data=params) + serializer.is_valid(raise_exception=True) + filters = serializer.validated_data + query = Log.objects.filter(menu='接入控制', operate__in=['允许访问', '拒绝访问']) + outcome = filters.get('result') or ('denied' if filters.get('denied') else '') + if outcome: + query = query.filter(status=403 if outcome == 'denied' else 200) + for key, lookup in [('start_time', 'create_time__gte'), ('end_time', 'create_time__lte'), + ('ip', 'ip_address__icontains'), ('device_type', 'details__device_type'), + ('reason', 'details__reason__icontains')]: + if filters.get(key): + query = query.filter(**{lookup: filters[key]}) + total = query.count() + size = filters['page_size'] + page = min(filters['page'], max(1, (total + size - 1) // size)) + rows = query.order_by('-create_time', '-id')[(page - 1) * size:page * size] + return {'total': total, 'current': page, 'page_size': size, 'records': [ + {'id': str(item.id), 'time': item.create_time, 'status': item.status, + 'ip': item.ip_address, 'target': (item.operation_object or {}).get('target'), + 'device_type': (item.details or {}).get('device_type', '未知'), + 'reason': (item.details or {}).get('reason', '')} + for item in rows]} diff --git a/apps/system_manage/ai_hub.py b/apps/system_manage/ai_hub.py index 5047c9e9f..2fca5c8a9 100644 --- a/apps/system_manage/ai_hub.py +++ b/apps/system_manage/ai_hub.py @@ -1,6 +1,6 @@ """AI hub business operations. Secrets never cross the browser boundary.""" from urllib.parse import urlsplit -from django.db import transaction +from django.db import connection, transaction from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact @@ -12,8 +12,8 @@ KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService, def admin(user): - from common.constants.permission_constants import RoleConstants - return user.role == str(RoleConstants.ADMIN.value) + from system_manage.platform_access import is_admin + return is_admin(user) def require_admin(user): @@ -26,26 +26,53 @@ def visible(query, user): if admin(user): return query if query.model in (AIParserResource, AIService): + from system_manage.platform_access import grants_for, expand_actions + kind = 'PARSER' if query.model == AIParserResource else 'SERVICE' + granted = {g.target for g in grants_for(user).filter(resource_type=kind) if 'VIEW' in expand_actions(g.actions)} ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id) - or str(user.id) in obj.allowed_users] + or str(user.id) in obj.allowed_users or str(obj.id) in granted] return query.filter(id__in=ids) return query.filter(owner_id=user.id) -def one(kind, pk, workspace, user, edit=False): +def one(kind, pk, workspace, user, edit=False, lock=False): from rest_framework.serializers import UUIDField pk = UUIDField().run_validation(pk) model = KINDS.get(kind) if not model: raise NotFound() - obj = visible(model.objects.filter(workspace_id=workspace), user).filter(id=pk).first() + query = model.objects.filter(workspace_id=workspace, id=pk) + if lock: + query = query.select_for_update() + obj = visible(query, user).first() if not obj: raise NotFound('记录不存在或无权访问') if edit and not admin(user) and str(obj.owner_id) != str(user.id): - raise PermissionDenied('无权修改') + from system_manage.platform_access import resource_actions + resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind) + if not resource_type or 'EDIT' not in resource_actions(user, resource_type, obj.id, workspace): + raise PermissionDenied('无权修改') return obj +def require_use(user, obj): + if admin(user) or str(obj.owner_id) == str(user.id) or str(user.id) in obj.allowed_users: + return + from system_manage.platform_access import resource_actions + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + if 'USE' not in resource_actions(user, kind, obj.id, obj.workspace_id): + raise PermissionDenied('未授权使用该资源') + + +def require_resource_action(user, obj, action): + if admin(user) or str(obj.owner_id) == str(user.id): + return + from system_manage.platform_access import resource_actions + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + if action not in resource_actions(user, kind, obj.id, obj.workspace_id): + raise PermissionDenied('无权操作该资源') + + def text(value, label, required=True, limit=MAX_TEXT): if not isinstance(value, str) or len(value) > limit or (required and not value.strip()): raise ValidationError(f'{label}无效') @@ -83,6 +110,8 @@ def authorized_models(workspace, user): grants = WorkspaceUserResourcePermission.objects.filter( workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL) ids = [g.target for g in grants if 'VIEW' in g.permission_list] + from system_manage.platform_access import grants_for, expand_actions + ids.extend(g.target for g in grants_for(user).filter(resource_type='MODEL') if 'USE' in expand_actions(g.actions)) query = query.filter(id__in=ids) return query @@ -96,8 +125,18 @@ def model_for(workspace, user, model_id): return obj -def serialize(obj, detail=False): +def serialize(obj, detail=False, user=None): data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()} + if user is not None: + data['can_delete'] = admin(user) or str(obj.owner_id) == str(user.id) + if user is not None and isinstance(obj, (AIParserResource, AIService)): + from system_manage.platform_access import resource_actions, ACTIONS + kind = 'PARSER' if isinstance(obj, AIParserResource) else 'SERVICE' + actions = set(ACTIONS) if admin(user) or str(obj.owner_id) == str(user.id) else resource_actions(user, kind, obj.id, obj.workspace_id) + if str(user.id) in obj.allowed_users: + actions |= {'VIEW', 'USE'} + data['actions'] = sorted(actions) + data['can_delete'] = 'EDIT' in actions if isinstance(obj, AIParserResource): data.update(enabled=obj.enabled, allowed_users=obj.allowed_users, options=obj.options, credential_configured=bool(obj.credential)) @@ -122,7 +161,13 @@ def serialize(obj, detail=False): def parser_save(workspace, user, data, obj=None): - require_admin(user) + if obj is None: + require_admin(user) + else: + if set(data) - {'allowed_users'}: + require_resource_action(user, obj, 'EDIT') + if 'allowed_users' in data: + require_resource_action(user, obj, 'AUTHORIZE') endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/') parsed = urlsplit(endpoint) if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment: @@ -141,17 +186,8 @@ def parser_save(workspace, user, data, obj=None): if 'allowed_users' in data: obj.allowed_users = users(data['allowed_users']) options = data.get('options', obj.options) - if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier'}: - raise ValidationError('解析参数无效') - if options.get('protocol', 'legacy') not in ('legacy', 'v1') or options.get('tier', 'standard') not in ('flash', 'basic', 'standard', 'advanced'): - raise ValidationError('解析接口或档位无效') - if options.get('backend', 'pipeline') not in ('pipeline', 'hybrid-auto-engine', 'vlm-auto-engine'): - raise ValidationError('不支持的解析后端') - if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'): - raise ValidationError('识别语言无效') - if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options): - raise ValidationError('解析参数无效') - obj.options = options + from system_manage.parsers.catalog import validate_options + obj.options = validate_options(options, endpoint) obj.save() return obj @@ -166,10 +202,9 @@ def parser_test(parser): if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password: raise ValidationError('解析服务地址无效') base_path = parsed.path - for suffix in ('/v1', '/api'): - if base_path.endswith(suffix): - base_path = base_path[:-len(suffix)] - break + if parser.options.get('protocol') == 'v1': + if base_path.endswith('/v1'): + base_path = base_path[:-3] url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', '')) headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} started = time.monotonic() @@ -183,13 +218,15 @@ def parser_test(parser): except ValueError as exc: raise ValidationError('解析服务未返回有效接口定义') from exc paths = data.get('paths') if isinstance(data, dict) else None - v1 = parser.options.get('protocol') == 'v1' or parsed.path.endswith('/v1') - suffix = '/tasks' if v1 else '/file_parse' - if not isinstance(paths, dict) or not any(path.endswith(suffix) and 'post' in methods - for path, methods in paths.items() if isinstance(methods, dict)): - raise ValidationError('未识别到对应的解析接口') + v1 = parser.options.get('protocol') == 'v1' + required = ('/v1/uploads', '/v1/parse/jobs') if v1 else ('/file_parse',) + if not isinstance(paths, dict) or not all(any(path.endswith(suffix) and 'post' in methods + for path, methods in paths.items() if isinstance(methods, dict)) for suffix in required): + raise ValidationError('服务接口与所选 MinerU 版本不匹配') + # Existence of the API does not prove that a local GPU/model backend is ready. return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000), - 'message': '解析接口连接正常'} + 'api_version': data.get('info', {}).get('version', ''), 'message': '解析接口连接正常'} + def validate_nodes(workspace, user, nodes): @@ -205,17 +242,31 @@ def validate_nodes(workspace, user, nodes): kind = node['type'] clean = {'type': kind} if kind == 'parse': - parser = one('parsers', node.get('resource_id'), workspace, user) + if node.get('processor') == 'builtin': + result.append({'type': 'parse', 'processor': 'builtin'}) + continue + parser = one('parsers', node.get('resource_id'), workspace, user, lock=connection.in_atomic_block) + require_use(user, parser) if not parser.enabled: raise ValidationError('解析服务已停用') clean['resource_id'] = str(parser.id) if kind == 'organize': clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id) clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000) - temperature = node.get('temperature', 0.2) - if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: - raise ValidationError('温度无效') - clean['temperature'] = temperature + temperature = node.get('temperature') + if temperature is not None: + if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: + raise ValidationError('温度无效') + clean['temperature'] = temperature + # Absent parameters fall back to the per-model settings in model management. + for field, low, high in ( + ('context_tokens', 4096, 262144), ('output_tokens', 256, 32768), + ('safety_tokens', 256, 8192), ('timeout', 30, 600)): + value = node.get(field) + if value is not None: + clean[field] = integer(value, low, high, '模型参数') + if clean.get('output_tokens', 4096) + clean.get('safety_tokens', 1024) >= clean.get('context_tokens', 32768): + raise ValidationError('输出与预留长度不能超过模型上下文') result.append(clean) return result @@ -224,7 +275,7 @@ def validate_nodes(workspace, user, nodes): def flow_create(workspace, user, data): nodes = validate_nodes(workspace, user, data.get('nodes')) # Every save is immutable; published services always reference a fixed row. - previous = one('flows', data['previous_id'], workspace, user, edit=True) if data.get('previous_id') else None + previous = one('flows', data['previous_id'], workspace, user, edit=True, lock=True) if data.get('previous_id') else None return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name'), '名称', limit=128), nodes=nodes, revision=previous.revision + 1 if previous else 1) @@ -233,7 +284,7 @@ def flow_create(workspace, user, data): @transaction.atomic def service_create(workspace, user, data): require_admin(user) - flow = one('flows', data.get('flow_id'), workspace, user) + flow = one('flows', data.get('flow_id'), workspace, user, lock=True) validate_nodes(workspace, user, flow.nodes) return AIService.objects.create(workspace_id=workspace, owner_id=user.id, name=text(data.get('name'), '名称', limit=128), flow=flow, @@ -245,6 +296,8 @@ def make_snapshot(workspace, user, nodes): clean = validate_nodes(workspace, user, nodes) for node in clean: if node['type'] == 'parse': + if node.get('processor') == 'builtin': + continue parser = one('parsers', node['resource_id'], workspace, user) node['options'] = dict(parser.options) node['resource_version'] = parser.updated_at.isoformat() @@ -255,7 +308,7 @@ def make_snapshot(workspace, user, nodes): @transaction.atomic -def submit(workspace, user, data, upload=None, service=None): +def submit(workspace, user, data, upload=None, service=None, request=None): from users.models import User current_user = User.objects.select_for_update().get(id=user.id) if not current_user.is_active: @@ -263,12 +316,16 @@ def submit(workspace, user, data, upload=None, service=None): if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10: raise ValidationError('待处理任务已达上限') if service: + service = one('services', service.pk, workspace, user, lock=True) + require_use(user, service) if not service.enabled: raise ValidationError('服务已停用') nodes = service.flow.nodes else: - nodes = one('flows', data['flow_id'], workspace, user).nodes if data.get('flow_id') else data.get('nodes') + nodes = one('flows', data['flow_id'], workspace, user, lock=True).nodes if data.get('flow_id') else data.get('nodes') snapshot = make_snapshot(workspace, user, nodes) + from system_manage.access_control import enforce_task + enforce_task(request, service.id if service else None, snapshot) markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT) if upload and upload.size > MAX_FILE: raise ValidationError('文件不能超过20MB') @@ -303,7 +360,7 @@ def artifact_save(workspace, user, data, obj=None): obj.history = history obj.revision += 1 else: - task = one('tasks', data['task_id'], workspace, user) if data.get('task_id') else None + task = one('tasks', data['task_id'], workspace, user, lock=True) if data.get('task_id') else None if task and task.status != 'SUCCESS': raise ValidationError('任务尚未成功') obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task, @@ -323,40 +380,43 @@ def artifact_save(workspace, user, data, obj=None): return obj +@transaction.atomic +def delete_record(kind, pk, workspace, user): + obj = one(kind, pk, workspace, user, edit=True, lock=True) + if kind == 'parsers': + reference = [{'type': 'parse', 'resource_id': str(obj.id)}] + if AIFlow.objects.filter(workspace_id=workspace, nodes__contains=reference).exists(): + raise ValidationError('解析服务已被流程引用,请先删除相关流程') + if AITask.objects.filter(workspace_id=workspace, status__in=['PENDING', 'RUNNING'], + snapshot__nodes__contains=reference).exists(): + raise ValidationError('解析服务有未完成任务,请稍后删除') + elif kind == 'flows': + if AIService.objects.filter(flow=obj).exists(): + raise ValidationError('流程已被服务引用,请先删除相关服务') + elif kind == 'services': + if AITask.objects.filter(service=obj, status__in=['PENDING', 'RUNNING']).exists(): + raise ValidationError('服务有未完成任务,请稍后删除') + elif kind == 'tasks': + if obj.status in ('PENDING', 'RUNNING'): + raise ValidationError('待处理或运行中的任务不能删除') + elif kind != 'artifacts': + raise ValidationError('资源类型无效') + # Historical task snapshots and saved artifacts remain independent records. + resource_type = {'parsers': 'PARSER', 'services': 'SERVICE'}.get(kind) + if resource_type: + from system_manage.models.platform_access import PlatformGrant + PlatformGrant.objects.filter(workspace_id=workspace, resource_type=resource_type, + target=str(obj.id)).delete() + obj.delete() + + def parse_file(parser, name, content, options=None, progress_callback=None): import requests from common.utils.rsa_util import rsa_long_decrypt opts = options if options is not None else parser.options headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} - if opts.get('protocol') == 'v1' or parser.endpoint.rstrip('/').endswith('/v1'): - from system_manage.ai_hub_parser_v1 import parse - return parse(parser, name, content, opts, headers, progress_callback=progress_callback) - try: - response = requests.post(parser.endpoint + '/file_parse', files={'files': (name, content)}, headers=headers, - data={'backend': opts.get('backend', 'pipeline'), 'lang_list': opts.get('language', 'ch'), - 'parse_method': 'ocr' if opts.get('ocr') else 'auto', 'return_md': 'true', - 'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true', - 'formula_enable': str(opts.get('formula', True)).lower(), - 'table_enable': str(opts.get('table', True)).lower()}, timeout=(10, 300), allow_redirects=False) - except Exception as exc: - raise ValidationError('解析服务连接失败或超时') from exc - if not 200 <= response.status_code < 300: - raise ValidationError('解析服务调用失败') - try: - payload = response.json() - except ValueError as exc: - raise ValidationError('解析服务返回格式无效') from exc - if not isinstance(payload, dict): - raise ValidationError('解析服务返回格式无效') - entries = payload.get('results', {}) - if not isinstance(entries, dict) or len(entries) != 1: - raise ValidationError('解析结果格式无效') - result = next(iter(entries.values())) - if not isinstance(result, dict): - raise ValidationError('解析结果格式无效') - markdown = text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT) - return {'markdown': markdown, 'source': result.get('middle_json', {}), 'parser_id': str(parser.id), - 'images': result.get('images', {})} + from system_manage.parsers import parse + return parse(parser, name, content, opts, headers, progress_callback=progress_callback) def markdown_tree(markdown): diff --git a/apps/system_manage/ai_hub_knowledge.py b/apps/system_manage/ai_hub_knowledge.py index 10458078e..e42e3d5f8 100644 --- a/apps/system_manage/ai_hub_knowledge.py +++ b/apps/system_manage/ai_hub_knowledge.py @@ -29,6 +29,10 @@ def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter parser_id = (knowledge.meta or {}).get('parser_id') if not parser_id: return None + from system_manage.parsers import MINERU_INPUTS, extension + if extension(file.name) not in MINERU_INPUTS: + # Preserve existing knowledge converters, segmentation and attachment ownership. + return None if user is None: raise PermissionDenied('解析服务调用需要用户身份') from system_manage.ai_hub_worker import execute @@ -45,10 +49,34 @@ def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter raise ValidationError(task.error or '解析失败') from django.core.files.uploadedfile import SimpleUploadedFile from common.handle.impl.text.text_split_handle import TextSplitHandle - content = task.result['markdown'].encode('utf-8') + content = knowledge_images(task.result['markdown'], workspace, knowledge_id, user).encode('utf-8') converted = SimpleUploadedFile(file.name + '.md', content) result = TextSplitHandle().handle(converted, patterns, with_filter, limit, lambda _: content, lambda _: None, overlap) result['name'] = file.name return result, {'resource_id': str(parser_id), 'task_id': str(task.id), 'resource_version': task.snapshot['nodes'][0]['resource_version']} + + +def knowledge_images(markdown, workspace, knowledge_id, user): + """Materialize parser images as knowledge attachments, with existing reader permissions and retention.""" + import mimetypes + import uuid + from knowledge.models import File, FileSourceType + from system_manage.ai_hub_assets import ASSET, image_bytes + replacements = {} + for match in ASSET.finditer(markdown): + if match[0] in replacements: + continue + raw, meta = image_bytes(workspace, user, match[2]) + file_id = uuid.uuid5(uuid.NAMESPACE_URL, f'knowledge-parser:{knowledge_id}:{match[2]}') + if not File.objects.filter(pk=file_id).exists(): + suffix = mimetypes.guess_extension(meta['mime']) or '.jpg' + attachment = File(id=file_id, file_name='parsed-image' + suffix, + source_type=FileSourceType.KNOWLEDGE, source_id=str(knowledge_id), + meta={'knowledge_id': str(knowledge_id), 'parser_asset_id': match[2]}) + attachment.save(raw) + replacements[match[0]] = './oss/file/' + str(file_id) + for before, after in replacements.items(): + markdown = markdown.replace(before, after) + return markdown diff --git a/apps/system_manage/ai_hub_organize.py b/apps/system_manage/ai_hub_organize.py new file mode 100644 index 000000000..d3db06b77 --- /dev/null +++ b/apps/system_manage/ai_hub_organize.py @@ -0,0 +1,111 @@ +"""Bounded hierarchical organization; source and completed chunks survive failures.""" +import re +import time + + +def token_bound(text): + # A conservative UTF-8 byte bound avoids assuming one tokenizer for all providers. + return len(text.encode('utf-8')) + + +def split_markdown(text, budget): + if budget < 256: + raise ValueError('模型上下文预算不足,请调大上下文或减少输出长度') + chunks, current = [], '' + blocks = re.split(r'(?m)(?=^#{1,6} )', text) + for block in blocks: + while block: + if token_bound(current + block) <= budget: + current += block + break + if current: + chunks.append(current); current = '' + if token_bound(block) <= budget: + current = block + break + lo, hi = 1, min(len(block), budget) + while lo < hi: + mid = (lo + hi + 1) // 2 + if token_bound(block[:mid]) <= budget: + lo = mid + else: + hi = mid - 1 + end = lo + newline = block.rfind('\n', 0, end) + if newline > end // 2: + end = newline + 1 + # Never split an image placeholder between requests. + for match in re.finditer(r'AIHUBIMAGE\s*\d+\s*TOKEN', block[:end + 64]): + if match.start() < end < match.end(): + end = match.start() + break + if not end: + raise ValueError('图片占位符超过模型输入预算') + chunks.append(block[:end]); block = block[end:] + if current: + chunks.append(current) + return chunks + + +def organize(markdown, prompt, invoke, *, context_tokens=32768, output_tokens=4096, + safety_tokens=1024, progress=None, checkpoint=None): + budget = context_tokens - output_tokens - safety_tokens - token_bound(prompt) - 512 + chunks = split_markdown(markdown, budget) + if len(chunks) > 128: + raise ValueError('文档分块过多,请调大模型上下文或拆分文档') + calls = 0 + + def call(content, instruction): + nonlocal calls + calls += 1 + if calls > 160: + raise ValueError('整理批次过多,请拆分文档') + if token_bound(content) + token_bound(instruction) > context_tokens - output_tokens - safety_tokens: + raise ValueError('模型输入超过上下文预算') + for attempt in range(3): + try: + response = invoke(instruction, content) + break + except Exception as exc: + if attempt == 2: + raise ValueError('模型调用失败,已保留原稿和完成的分块,请稍后重试') from exc + time.sleep(attempt + 1) + metadata = getattr(response, 'response_metadata', {}) + if isinstance(metadata, dict) and metadata.get('finish_reason') in ('length', 'max_tokens'): + raise ValueError('模型输出被截断,请到模型管理调大该模型的输出上限,或减少整理内容') + result = getattr(response, 'content', response) + if isinstance(result, list): + result = ''.join(item.get('text', '') for item in result if isinstance(item, dict)) + if not isinstance(result, str) or not result.strip(): + raise ValueError('模型未返回有效整理内容') + return result.strip() + + outputs = [] + chunk_prompt = prompt + '\n这是文档的一个分块,保留事实及标题关系,精简重复内容。' + for index, chunk in enumerate(chunks, 1): + if progress: + progress({'stage': f'智能整理分块 {index}/{len(chunks)}', 'percent': (index - 1) * 70 // len(chunks)}) + outputs.append(call(chunk, prompt if len(chunks) == 1 else chunk_prompt)) + if checkpoint: + checkpoint(list(outputs)) + if len(outputs) == 1: + return outputs[0] + merge_prompt = prompt + '\n合并以下局部大纲,统一标题层级,压缩重复内容;保留事实、数字和图片占位符。' + previous_size = None + for round_index in range(4): + joined = '\n\n'.join(outputs) + groups = split_markdown(joined, budget) + if previous_size is not None and token_bound(joined) >= previous_size and len(groups) > 1: + raise ValueError('合并结果未收敛,已保留完成的分块,请缩小文档或调整提示词') + previous_size = token_bound(joined) + merged = [] + for index, group in enumerate(groups, 1): + if progress: + progress({'stage': f'合并大纲 {round_index + 1}:{index}/{len(groups)}', 'percent': 70 + round_index * 6}) + merged.append(call(group, merge_prompt)) + if checkpoint: + checkpoint(merged) + if len(merged) == 1: + return merged[0] + outputs = merged + raise ValueError('合并批次超过限制,已保留完成的分块') diff --git a/apps/system_manage/ai_hub_parser_v1.py b/apps/system_manage/ai_hub_parser_v1.py index 4ee7128fc..ad8faa9ea 100644 --- a/apps/system_manage/ai_hub_parser_v1.py +++ b/apps/system_manage/ai_hub_parser_v1.py @@ -1,59 +1,165 @@ -"""MinerU v1 task API adapter. Only follows paths built from the registered endpoint.""" +"""MinerU 4 V1 upload/job/file protocol, independent of convenience routes.""" +import base64 +import hashlib +import io +import json +import mimetypes import re import time +import zipfile +from pathlib import PurePosixPath +from urllib.parse import urljoin, urlsplit import requests from rest_framework.exceptions import ValidationError +MAX_OUTPUT = 150 * 1024 * 1024 -def _json(response): - if response.status_code not in (200, 201, 202): - raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + +def origin(url): + p = urlsplit(url) + if p.scheme not in ('http', 'https') or not p.hostname or p.username or p.password: + raise ValidationError('解析服务返回地址无效') + return p.scheme.lower(), p.hostname.lower(), p.port or (443 if p.scheme == 'https' else 80) + + +def identifier(value): + if not isinstance(value, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,200}', value): + raise ValidationError('解析服务返回资源 ID 无效') + return value + + +def read(response): + chunks, size = [], 0 try: - body = response.json() - except ValueError as exc: + for chunk in response.iter_content(65536): + size += len(chunk) + if size > MAX_OUTPUT: + raise ValidationError('解析结果超过大小限制') + chunks.append(chunk) + finally: + response.close() + return b''.join(chunks) + + +def call(method, url, headers, expect_json=True, **kwargs): + response = requests.request(method, url, headers=headers, timeout=(10, 30), allow_redirects=False, + stream=True, **kwargs) + if response.status_code not in (200, 201, 202, 204): + response.close() + raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + raw = read(response) + if not expect_json or response.status_code == 204 or not raw: + return {} + try: + result = json.loads(raw) + except (ValueError, UnicodeDecodeError) as exc: raise ValidationError('解析服务返回格式无效') from exc - if not isinstance(body, dict): + if not isinstance(result, dict): raise ValidationError('解析服务返回格式无效') - return body + return result + + +def download(url, base, headers): + for _ in range(4): + response = requests.get(url, headers=headers if origin(url) == origin(base) else {}, + timeout=(10, 30), allow_redirects=False, stream=True) + if response.status_code in (301, 302, 303, 307, 308): + target = response.headers.get('Location', '') + response.close() + if not target: + raise ValidationError('解析结果跳转地址无效') + url = urljoin(url, target) + origin(url) + continue + if response.status_code != 200: + response.close() + raise ValidationError(f'获取解析结果失败(HTTP {response.status_code})') + return read(response) + raise ValidationError('解析结果跳转次数过多') + + +def unpack(raw): + images = {} + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + if sum(i.file_size for i in archive.infolist()) > 200 * 1024 * 1024: + raise ValidationError('解析结果解压后超过大小限制') + for item in archive.infolist(): + path = PurePosixPath(item.filename) + if '..' in path.parts or path.is_absolute(): + raise ValidationError('解析结果路径无效') + mime = mimetypes.guess_type(item.filename)[0] or '' + if mime.startswith('image/'): + images[item.filename] = 'data:' + mime + ';base64,' + base64.b64encode(archive.read(item)).decode() + except (zipfile.BadZipFile, RuntimeError) as exc: + raise ValidationError('解析结果压缩包无效') from exc + return images def parse(parser, name, content, options, headers, progress_callback=None): from system_manage.ai_hub import text, MAX_INLINE_TEXT + from system_manage.ai_hub_progress import parser_progress endpoint = parser.endpoint.rstrip('/') + base = endpoint if endpoint.endswith('/v1') else endpoint + '/v1' + root = base[:-3] deadline = time.monotonic() + 300 try: - response = requests.post(endpoint + '/tasks', files={'files': (name, content)}, headers=headers, - params=[('tier', options.get('tier', 'standard')), ('ocr_mode', 'ocr' if options.get('ocr') else 'auto'), - ('output_formats', 'markdown'), ('output_formats', 'middle_json')], - timeout=(10, 30), allow_redirects=False) - task = _json(response) - from system_manage.ai_hub_progress import parser_progress - task_id = task.get('task_id', '') - if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id): - raise ValidationError('解析服务未返回有效任务ID') - if progress_callback: - progress_callback(parser_progress(task)) - while task.get('status') in ('queued', 'running'): - if time.monotonic() >= deadline: - raise ValidationError('解析等待超时,后台任务ID:' + task_id) - time.sleep(2) - task = _json(requests.get(endpoint + '/tasks/' + task_id, headers=headers, - timeout=(10, 20), allow_redirects=False)) + upload = call('POST', base + '/uploads', headers, json={ + 'filename': name, 'bytes': len(content), 'mime_type': mimetypes.guess_type(name)[0] or 'application/octet-stream', + 'purpose': 'parse', 'sha256sum': hashlib.sha256(content).hexdigest()}) + if upload.get('status') == 'pending': + uid = identifier(upload.get('id')) + target = urljoin(root + '/', upload.get('upload_url', '')) + if not upload.get('upload_url') or upload.get('upload_method', 'PUT') != 'PUT': + raise ValidationError('解析上传协议无效') + origin(target) + extra = upload.get('upload_headers') or {} + if not isinstance(extra, dict) or any(not isinstance(k, str) or not isinstance(v, str) or '\n' in k+v or '\r' in k+v for k,v in extra.items()): + raise ValidationError('解析上传请求头无效') + # Never forward the registered parser credential to object storage. + auth = {**extra, **headers} if origin(target) == origin(base) else extra + call('PUT', target, auth, expect_json=False, data=content) + upload = call('POST', base + '/uploads/' + uid + '/complete', headers) + if upload.get('status') != 'completed': + raise ValidationError('解析文件上传未完成') + fid = identifier((upload.get('file') or {}).get('id')) + job = call('POST', base + '/parse/jobs', headers, json={ + 'files': [{'source': {'type': 'file_id', 'file_id': fid}}], + 'tier': options.get('tier', 'standard'), 'ocr_mode': 'ocr' if options.get('ocr') else 'auto', + 'output_formats': ['markdown', 'middle_json', 'zip']}) + jid = identifier(job.get('job_id')) + while True: + state = job.get('status') + if state not in ('queued', 'running', 'completed', 'partial', 'failed', 'canceled'): + raise ValidationError('解析任务状态无效') if progress_callback: - progress_callback(parser_progress(task)) - if task.get('status') != 'completed': - raise ValidationError('解析任务未成功,后台任务ID:' + task_id) - result = _json(requests.get(endpoint + '/tasks/' + task_id + '/result', headers=headers, - timeout=(10, 30), allow_redirects=False)) + progress_callback(parser_progress(job)) + if state not in ('queued', 'running'): + break + if time.monotonic() >= deadline: + raise ValidationError('解析等待超时,后台任务 ID:' + jid) + time.sleep(2) + job = call('GET', base + '/parse/jobs/' + jid, headers) + files = job.get('files') + if state != 'completed' or not isinstance(files, list) or len(files) != 1 or files[0].get('status') != 'completed': + raise ValidationError('解析任务未成功,后台任务 ID:' + jid) + outputs = files[0].get('output_files') or {} + def artifact(kind): + ref = outputs.get(kind) + if not isinstance(ref, dict): + raise ValidationError('解析结果缺少 ' + kind) + return download(base + '/files/' + identifier(ref.get('file_id')) + '/content', base, headers) + markdown = artifact('markdown').decode('utf-8') + middle = json.loads(artifact('middle_json')) + images = unpack(artifact('zip')) if 'zip' in outputs else {} + # ZIP outputs can wrap a document directory; include relative image paths. + for path, value in list(images.items()): + parts = PurePosixPath(path).parts + if 'images' in parts: + images['/'.join(parts[parts.index('images'):])] = value + return {'markdown': text(markdown, '解析结果', limit=MAX_INLINE_TEXT), 'parser_id': str(parser.id), + 'images': images, 'source': {'task_id': jid, 'middle_json': middle}} except requests.RequestException as exc: raise ValidationError('解析服务连接失败或超时') from exc - files = result.get('files', []) - if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) or files[0].get('status') != 'completed': - raise ValidationError('解析文件结果无效') - output = files[0].get('content', {}) - if not isinstance(output, dict): - raise ValidationError('解析文件内容无效') - markdown = text(output.get('markdown'), '解析结果', limit=MAX_INLINE_TEXT) - return {'markdown': markdown, 'parser_id': str(parser.id), 'images': output.get('images', {}), - 'source': {'task_id': task_id, 'parse': files[0].get('parse'), - 'middle_json': output.get('middle_json', {})}} + except (ValueError, UnicodeDecodeError) as exc: + raise ValidationError('解析结果内容无效') from exc diff --git a/apps/system_manage/ai_hub_worker.py b/apps/system_manage/ai_hub_worker.py index eaff5de81..ce1f5cf7c 100644 --- a/apps/system_manage/ai_hub_worker.py +++ b/apps/system_manage/ai_hub_worker.py @@ -23,8 +23,12 @@ def execute(task): user = User.objects.get(pk=task.owner_id) if not user.is_active: raise ValueError('用户已停用') + from system_manage.platform_access import module_list + if 'mindmap' not in module_list(user): + raise ValueError('未授权使用思维导图') if task.service_id: service = hub.one('services', task.service_id, task.workspace_id, user) + hub.require_use(user, service) if not service.enabled: raise ValueError('服务已停用') markdown = task.input_data.get('markdown', '') @@ -39,15 +43,23 @@ def execute(task): task.steps = steps task.save(update_fields=['steps', 'updated_at']) if node['type'] == 'parse': - parser = hub.one('parsers', node['resource_id'], task.workspace_id, user) - if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']: - raise ValueError('解析服务已停用或版本发生变化,请重新提交') + parser = None + if node.get('processor') != 'builtin': + parser = hub.one('parsers', node['resource_id'], task.workspace_id, user) + hub.require_use(user, parser) + if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']: + raise ValueError('解析服务已停用或版本发生变化,请重新提交') def report_progress(progress): step['progress'] = progress AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update( steps=steps, updated_at=timezone.now()) report_progress({'stage': '上传文档到解析服务', 'percent': None}) - parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) + if parser is None: + from system_manage.parsers.native import parse + report_progress({'stage': '使用内置处理器解析文档', 'percent': None}) + parsed = parse(task.file_name, bytes(task.file_data)) + else: + parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) from system_manage.ai_hub_assets import asset_lock, normalize with asset_lock(): markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果') @@ -66,17 +78,33 @@ def execute(task): # Bound input; never silently truncate the original document. from system_manage.ai_hub_assets import protect_images, restore_images model_markdown, image_tokens = protect_images(markdown) - if len(model_markdown) > 60000: - raise ValueError('文档超过智能整理长度限制,请分批处理') try: - instance = get_model(model, **{**get_model_default_params(model), - 'temperature': node.get('temperature', 0.2), 'timeout': 180}) + # Per-model settings from model management win; task nodes override only when set. + defaults = get_model_default_params(model) + temperature = node.get('temperature') + if temperature is None: + temperature = defaults.get('temperature', 0.2) + max_tokens = node.get('output_tokens') + if max_tokens is None: + max_tokens = defaults.get('max_tokens', 4096) + instance = get_model(model, **{**defaults, 'temperature': temperature, + 'max_tokens': max_tokens, 'timeout': node.get('timeout', 180)}) prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。' - response = instance.invoke([SystemMessage(content=prompt), HumanMessage(content=model_markdown)]) except Exception as exc: from rest_framework.exceptions import ValidationError raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc - markdown = restore_images(hub.text(response.content, '整理结果'), image_tokens) + from system_manage.ai_hub_organize import organize + def organize_progress(progress): + step['progress'] = progress + AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(steps=steps, updated_at=timezone.now()) + def checkpoint(outputs): + task.result['organize_chunks'] = outputs + task.save(update_fields=['result', 'updated_at']) + organized = organize(model_markdown, prompt, + lambda instruction, content: instance.invoke([SystemMessage(content=instruction), HumanMessage(content=content)]), + context_tokens=node.get('context_tokens', 32768), output_tokens=max_tokens, + safety_tokens=node.get('safety_tokens', 1024), progress=organize_progress, checkpoint=checkpoint) + markdown = restore_images(hub.text(organized, '整理结果'), image_tokens) else: tree = hub.markdown_tree(markdown) step['status'] = 'SUCCESS' diff --git a/apps/system_manage/document_export.py b/apps/system_manage/document_export.py new file mode 100644 index 000000000..073155bef --- /dev/null +++ b/apps/system_manage/document_export.py @@ -0,0 +1,305 @@ +"""Structured DOCX authoring and bounded, isolated LibreOffice conversions. + +No URL fetching, shell interpolation, macros, shared profiles, or database writes. +""" +import base64 +import contextlib +import io +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import time + +from bs4 import BeautifulSoup, NavigableString, Tag +from docx import Document +from docx.enum.text import WD_BREAK +from docx.oxml import OxmlElement +from docx.oxml.ns import qn +from docx.shared import Inches, Pt +from PIL import Image + +MAX_HTML = 24 * 1024 * 1024 +MAX_IMAGES = 16 * 1024 * 1024 +MIMES = {'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'pdf': 'application/pdf', 'rtf': 'application/rtf'} + + +class ExportError(ValueError): + pass + + +def _element(parent, name, **attrs): + child = OxmlElement(name) + for key, value in attrs.items(): + child.set(qn('w:' + key), str(value)) + parent.append(child) + return child + + +@contextlib.contextmanager +def conversion_slot(): + # A machine-wide file lock also bounds work across multiple Web processes. + path = Path(tempfile.gettempdir()) / 'maxkb-document-export.lock' + with path.open('a+b') as stream: + stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0) + deadline = time.monotonic() + 15 + while True: + try: + if os.name == 'nt': + import msvcrt + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except OSError: + if time.monotonic() > deadline: + raise ExportError('导出繁忙,请稍后重试') + time.sleep(.1) + try: + yield + finally: + if os.name == 'nt': + import msvcrt + stream.seek(0); msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + + +class DocumentBuilder: + def __init__(self, image_resolver=None, paper='A4'): + self.doc = Document() + self.resolve = image_resolver + self.image_bytes = 0 + self.image_count = 0 + section = self.doc.sections[0] + section.page_width = Inches(8.2677) + section.page_height = Inches(11.6929) + section.top_margin = section.bottom_margin = Inches(.75) + section.left_margin = section.right_margin = Inches(.8) + self.width = section.page_width - section.left_margin - section.right_margin + for name in ['Normal', 'Title', *['Heading ' + str(i) for i in range(1, 7)]]: + style = self.doc.styles[name] + style.font.name = 'Noto Sans CJK SC' + style.font.size = Pt(11 if name == 'Normal' else 22 if name == 'Title' else max(11, 19 - int(name[-1]))) + style.font.color.rgb = __import__('docx').shared.RGBColor(0, 0, 0) + fonts = style.element.get_or_add_rPr().rFonts + fonts.set(qn('w:eastAsia'), 'Noto Sans CJK SC') + style.paragraph_format.space_after = Pt(6) + style.paragraph_format.line_spacing = 1.25 + style.paragraph_format.widow_control = True + footer = section.footer.paragraphs[0] + footer.alignment = 1 + field = _element(footer._p, 'w:fldSimple', instr='PAGE') + _element(_element(field, 'w:r'), 'w:t').text = '1' + + def image(self, node, paragraph, width=None): + self.image_count += 1 + if self.image_count > 128: raise ExportError('单次导出图片不能超过128张') + src = node.get('src', '') + if src.startswith('data:image/'): + match = re.fullmatch(r'data:image/(png|jpeg|gif|webp|bmp);base64,([A-Za-z0-9+/=\s]+)', src) + if not match: + raise ExportError('图片格式无效') + try: + raw = base64.b64decode(match[2], validate=False) + except (ValueError, TypeError) as exc: + raise ExportError('图片格式无效') from exc + elif self.resolve: + raw = self.resolve(src) + else: + raise ExportError('图片未保存,请重新加载后导出') + self.image_bytes += len(raw) + if self.image_bytes > MAX_IMAGES: + raise ExportError('导出图片总大小不能超过16MB') + try: + with Image.open(io.BytesIO(raw)) as image: + if image.width * image.height > 40_000_000: + raise ExportError('图片尺寸过大') + image.load() + # DOCX/RTF and Office readers share a reliable raster baseline. + converted = io.BytesIO() + image.convert('RGBA' if image.mode == 'RGBA' else 'RGB').save(converted, format='PNG') + natural_width, natural_height = image.size + max_width = min(width or self.width, self.width) + requested_width = str(node.get('width', natural_width)) + pixels = int(requested_width) if requested_width.isdigit() and 0 < int(requested_width) < 10000 else natural_width + display_width = min(max_width, Inches(pixels / 96)) + display_width = min(display_width, Inches(8.8) * natural_width / max(1, natural_height)) + picture = paragraph.add_run().add_picture(io.BytesIO(converted.getvalue()), width=int(display_width)) + picture._inline.docPr.set('descr', node.get('alt', '')) + except ExportError: + raise + except Exception as exc: + raise ExportError('图片无法导出,请检查图片格式') from exc + + def inline(self, node, paragraph, formats=None, width=None): + formats = dict(formats or {}) + if isinstance(node, NavigableString): + run = paragraph.add_run(str(node)) + for name in ('bold', 'italic', 'underline', 'strike'): + if formats.get(name): + if name == 'strike': run.font.strike = True + else: setattr(run, name, True) + if formats.get('subscript'): run.font.subscript = True + if formats.get('superscript'): run.font.superscript = True + if formats.get('color'): + from docx.shared import RGBColor + run.font.color.rgb = RGBColor.from_string(formats['color']) + if formats.get('code'): + run.font.name = 'DejaVu Sans Mono'; run.font.size = Pt(9) + return + if not isinstance(node, Tag): + return + tag = node.name.lower() + if tag in ('script', 'style', 'iframe', 'object', 'svg', 'math'): + return + if tag == 'img': + self.image(node, paragraph, width); return + if tag == 'br': + paragraph.add_run().add_break(WD_BREAK.LINE); return + mapping = {'b': 'bold', 'strong': 'bold', 'i': 'italic', 'em': 'italic', 'u': 'underline', 's': 'strike', 'del': 'strike', 'code': 'code', 'sub': 'subscript', 'sup': 'superscript'} + if tag in mapping: formats[mapping[tag]] = True + color = re.search(r'(?:^|;)\s*color\s*:\s*#([0-9a-fA-F]{6})(?:;|$)', node.get('style', '')) + if color: formats['color'] = color[1] + if tag == 'a': + href = node.get('href', '') + if re.match(r'^(https?://|mailto:)', href, re.I): + from docx.opc.constants import RELATIONSHIP_TYPE as RT + hyperlink = _element(paragraph._p, 'w:hyperlink') + hyperlink.set(qn('r:id'), paragraph.part.relate_to(href, RT.HYPERLINK, is_external=True)) + # Build styled text with normal runs, then move them into the link. + start = len(paragraph.runs) + for child in node.children: self.inline(child, paragraph, formats, width) + for run in list(paragraph.runs)[start:]: hyperlink.append(run._r) + return + for child in node.children: + self.inline(child, paragraph, formats, width) + + def numbering(self, ordered, start=1): + numbering = self.doc.part.numbering_part.element + abstract_id = max([int(n.get(qn('w:abstractNumId'))) for n in numbering.findall(qn('w:abstractNum'))] + [-1]) + 1 + abstract = _element(numbering, 'w:abstractNum', abstractNumId=abstract_id) + _element(abstract, 'w:multiLevelType', val='multilevel') + for level in range(9): + lvl = _element(abstract, 'w:lvl', ilvl=level) + _element(lvl, 'w:start', val=start if level == 0 else 1) + _element(lvl, 'w:numFmt', val='decimal' if ordered else 'bullet') + _element(lvl, 'w:lvlText', val=f'%{level + 1}.' if ordered else '•') + _element(_element(lvl, 'w:pPr'), 'w:ind', left=360 * (level + 1), hanging=240) + num_id = max([int(n.get(qn('w:numId'))) for n in numbering.findall(qn('w:num'))] + [0]) + 1 + _element(_element(numbering, 'w:num', numId=num_id), 'w:abstractNumId', val=abstract_id) + return num_id + + def blocks(self, nodes, container=None, level=0, width=None): + container = self.doc if container is None else container + width = width or self.width + for node in nodes: + if isinstance(node, NavigableString): + if str(node).strip(): container.add_paragraph(str(node)) + continue + if not isinstance(node, Tag): continue + tag = node.name.lower() + if tag in ('script', 'style', 'iframe', 'object'): continue + if tag in ('ul', 'ol'): + num_id = self.numbering(tag == 'ol', min(100000, max(1, int(node.get('start', 1))))) + for item in node.find_all('li', recursive=False): + p = container.add_paragraph() + props = _element(p._p.get_or_add_pPr(), 'w:numPr') + _element(props, 'w:ilvl', val=min(level, 8)); _element(props, 'w:numId', val=num_id) + for child in item.children: + if isinstance(child, Tag) and child.name in ('ul', 'ol'): + self.blocks([child], container, level + 1, width) + else: self.inline(child, p, width=width) + continue + if tag == 'table': + rows = node.find_all('tr') + if len(rows) > 2000: raise ExportError('表格行数过多') + occupied = set(); placements = []; columns = 0 + for row_index, row in enumerate(rows): + column = 0 + for cell in row.find_all(['td', 'th'], recursive=False): + while (row_index, column) in occupied: column += 1 + colspan = min(30, max(1, int(cell.get('colspan', 1)))) + rowspan = min(len(rows) - row_index, max(1, int(cell.get('rowspan', 1)))) + if column + colspan > 30: raise ExportError('表格列数不能超过30列') + for rr in range(row_index, row_index + rowspan): + for cc in range(column, column + colspan): + if (rr, cc) in occupied: raise ExportError('表格合并区域无效') + occupied.add((rr, cc)) + placements.append((row_index, column, rowspan, colspan, cell)) + column += colspan; columns = max(columns, column) + if not columns: continue + table = container.add_table(rows=len(rows), cols=columns) + table.style = 'Table Grid'; table.autofit = False + for column in table.columns: column.width = int(width / columns) + table_rows = list(table.rows) + # python-docx reconstructs the entire grid on every table.cell() + # call. Cache it once; placement targets never point inside a merged span. + grid = list(table._cells) + for index, row in enumerate(rows): + if row.find('th') is not None: _element(table_rows[index]._tr.get_or_add_trPr(), 'w:tblHeader') + _element(table_rows[index]._tr.get_or_add_trPr(), 'w:cantSplit') + for rr, cc, rowspan, colspan, cell in placements: + target = grid[rr * columns + cc] + if rowspan > 1 or colspan > 1: target = target.merge(grid[(rr + rowspan - 1) * columns + cc + colspan - 1]) + p = target.paragraphs[0] + alignment = re.search(r'text-align:\s*(left|center|right)', cell.get('style', '')) + if alignment: p.alignment = {'left':0, 'center':1, 'right':2}[alignment[1]] + for child in cell.children: self.inline(child, p, {'bold': cell.name == 'th'}, int(width / columns * colspan)) + container.add_paragraph(); continue + if tag in ('div', 'section', 'article', 'main', 'body', 'blockquote'): + before = len(container.paragraphs) + self.blocks(list(node.children), container, level, width) + if tag == 'blockquote': + for p in container.paragraphs[before:]: p.paragraph_format.left_indent = Inches(.25) + continue + if tag in ('thead', 'tbody', 'tfoot'): continue + p = container.add_paragraph() + if re.fullmatch('h[1-6]', tag): + p.style = 'Heading ' + tag[1]; p.paragraph_format.keep_with_next = True + if tag == 'pre': + p.paragraph_format.space_before = Pt(5) + shading = _element(p._p.get_or_add_pPr(), 'w:shd', fill='F5F7FA') + run = p.add_run(node.get_text()); run.font.name = 'DejaVu Sans Mono'; run.font.size = Pt(9) + elif tag == 'hr': + _element(_element(p._p.get_or_add_pPr(), 'w:pBdr'), 'w:bottom', val='single', sz=4, color='D9DDE4') + else: self.inline(node, p, width=width) + + def build(self, html): + soup = BeautifulSoup(html, 'html.parser') + if len(soup.find_all(True)) > 50000: raise ExportError('文档结构过于复杂,请拆分后导出') + self.blocks(list((soup.body or soup).children)) + result = io.BytesIO(); self.doc.save(result) + return result.getvalue() + + +def export_document(html, format_name='docx', image_resolver=None, paper='A4'): + if format_name not in MIMES: raise ExportError('不支持的导出格式') + if paper != 'A4': raise ExportError('仅支持 A4 纸张') + if not isinstance(html, str) or not html.strip() or len(html.encode('utf-8')) > MAX_HTML: + raise ExportError('导出内容为空或超过24MB') + raw = DocumentBuilder(image_resolver, paper).build(html) + if format_name == 'docx': return raw, MIMES[format_name] + command = shutil.which('libreoffice') or shutil.which('soffice') + if command is None: raise ExportError('文档转换组件不可用,请联系管理员或下载 DOCX') + with conversion_slot(), tempfile.TemporaryDirectory(prefix='maxkb-export-') as directory: + root = Path(directory) + (root / 'document.docx').write_bytes(raw) + try: + completed = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(), + '--headless', '--convert-to', format_name, '--outdir', str(root), str(root / 'document.docx')], + stdin=subprocess.DEVNULL, capture_output=True, timeout=90, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ExportError('文档转换超时或失败,请下载 DOCX 后重试') from exc + result = root / ('document.' + format_name) + if completed.returncode or not result.is_file() or result.stat().st_size == 0: + raise ExportError('文档转换失败,请下载 DOCX 后重试') + output = result.read_bytes() + if format_name == 'pdf' and not output.startswith(b'%PDF-'): raise ExportError('PDF 输出无效') + if format_name == 'rtf' and not output.lstrip().startswith(b'{\\rtf'): raise ExportError('RTF 输出无效') + return output, MIMES[format_name] diff --git a/apps/system_manage/management/commands/unify_platform_roles.py b/apps/system_manage/management/commands/unify_platform_roles.py new file mode 100644 index 000000000..4981de4ed --- /dev/null +++ b/apps/system_manage/management/commands/unify_platform_roles.py @@ -0,0 +1,11 @@ +"""Import existing account roles into the unified role/member tables.""" +import json +from django.core.management.base import BaseCommand +from system_manage.platform_access import reconcile_builtin_roles + + +class Command(BaseCommand): + help = 'Backfill built-in platform roles; preserves custom roles and resource grants. No schema migration.' + + def handle(self, *args, **options): + self.stdout.write(json.dumps(reconcile_builtin_roles(), ensure_ascii=False)) diff --git a/apps/system_manage/migrations/0008_platform_access.py b/apps/system_manage/migrations/0008_platform_access.py new file mode 100644 index 000000000..749589e94 --- /dev/null +++ b/apps/system_manage/migrations/0008_platform_access.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.8 on 2026-10-23 06:05 + +import django.contrib.postgres.fields +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +EFFECTIVE_VIEW = """ +CREATE VIEW effective_resource_permission AS +SELECT id, workspace_id, user_id, auth_target_type, target, auth_type, + permission_list, create_time, update_time +FROM workspace_user_resource_permission +UNION ALL +SELECT g.id, g.workspace_id, u.id, g.resource_type::varchar(128), g.target, + 'RESOURCE_PERMISSION_GROUP'::varchar, + (ARRAY['VIEW']::varchar[] || CASE WHEN g.actions ? 'EDIT' + THEN ARRAY['MANAGE']::varchar[] ELSE ARRAY[]::varchar[] END), + g.updated_at, g.updated_at +FROM platform_grant g JOIN "user" u ON u.is_active = true +WHERE g.resource_type IN ('KNOWLEDGE','APPLICATION','MODEL','TOOL') + AND g.actions <> '[]'::jsonb + AND (g.expires_at IS NULL OR g.expires_at > statement_timestamp()) + AND NOT EXISTS (SELECT 1 FROM user_activation_application a WHERE a.user_id=u.id AND a.status <> 'APPROVED') + AND (g.subject_type='ALL' + OR (g.subject_type='USER' AND g.subject_id=u.id::text) + OR (g.subject_type='ROLE' AND EXISTS ( + SELECT 1 FROM platform_role r WHERE r.id::text=g.subject_id AND r.enabled + AND (r.is_default OR EXISTS (SELECT 1 FROM platform_role_member m WHERE m.role_id=r.id AND m.user_id=u.id))))) +""" + + +def seed_default(apps, schema_editor): + apps.get_model('system_manage', 'PlatformRole').objects.get_or_create( + id='00000000-0000-0000-0000-000000000001', + defaults={'name': '普通用户', 'description': '所有已激活用户自动关联', + 'is_default': True, 'modules': ['knowledge', 'mindmap']}) + + +class Migration(migrations.Migration): + + dependencies = [ + ('system_manage', '0007_ai_hub'), + ('users', '0002_user_activation_and_role_assignment'), + ] + + operations = [ + migrations.CreateModel( + name='EffectiveResourcePermission', + fields=[ + ('id', models.UUIDField(primary_key=True, serialize=False)), + ('workspace_id', models.CharField(max_length=128)), + ('user_id', models.UUIDField()), + ('auth_target_type', models.CharField(max_length=128)), + ('target', models.CharField(max_length=128)), + ('auth_type', models.CharField(max_length=64)), + ('permission_list', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), size=None)), + ('create_time', models.DateTimeField()), + ('update_time', models.DateTimeField()), + ], + options={ + 'db_table': 'effective_resource_permission', + 'managed': False, + }, + ), + migrations.CreateModel( + name='PlatformRole', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=64, unique=True)), + ('description', models.CharField(blank=True, default='', max_length=256)), + ('enabled', models.BooleanField(default=True)), + ('is_default', models.BooleanField(default=False)), + ('modules', models.JSONField(default=list)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'platform_role', + }, + ), + migrations.CreateModel( + name='PlatformGrant', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('subject_type', models.CharField(choices=[('ALL', '所有用户'), ('ROLE', '角色'), ('USER', '用户')], max_length=8)), + ('subject_id', models.CharField(blank=True, default='', max_length=64)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('resource_type', models.CharField(max_length=32)), + ('target', models.CharField(max_length=128)), + ('actions', models.JSONField(default=list)), + ('expires_at', models.DateTimeField(blank=True, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('granted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user')), + ], + options={ + 'db_table': 'platform_grant', + 'indexes': [models.Index(fields=['subject_type', 'subject_id'], name='platform_grant_subject_idx')], + 'constraints': [models.UniqueConstraint(fields=('workspace_id', 'subject_type', 'subject_id', 'resource_type', 'target'), name='unique_platform_grant')], + }, + ), + migrations.CreateModel( + name='PlatformRoleMember', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='system_manage.platformrole')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='platform_roles', to='users.user')), + ], + options={ + 'db_table': 'platform_role_member', + 'constraints': [models.UniqueConstraint(fields=('role', 'user'), name='unique_platform_role_member')], + }, + ), + migrations.RunSQL(EFFECTIVE_VIEW, 'DROP VIEW IF EXISTS effective_resource_permission'), + migrations.RunPython(seed_default, migrations.RunPython.noop), + ] diff --git a/apps/system_manage/models/__init__.py b/apps/system_manage/models/__init__.py index b5f3fd7d8..b55d2c582 100644 --- a/apps/system_manage/models/__init__.py +++ b/apps/system_manage/models/__init__.py @@ -13,3 +13,4 @@ from .chat_user import * from .user_menu_setting import * from .access_control_policy import * from .ai_hub import * +from .platform_access import * diff --git a/apps/system_manage/models/platform_access.py b/apps/system_manage/models/platform_access.py new file mode 100644 index 000000000..f433896bb --- /dev/null +++ b/apps/system_manage/models/platform_access.py @@ -0,0 +1,62 @@ +"""Platform-wide roles and additive grants in the default workspace.""" +import uuid +from django.db import models +from django.contrib.postgres.fields import ArrayField + + +class PlatformRole(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=64, unique=True) + description = models.CharField(max_length=256, blank=True, default='') + enabled = models.BooleanField(default=True) + is_default = models.BooleanField(default=False) + modules = models.JSONField(default=list) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'platform_role' + + +class PlatformRoleMember(models.Model): + role = models.ForeignKey(PlatformRole, on_delete=models.CASCADE, related_name='members') + user = models.ForeignKey('users.User', on_delete=models.CASCADE, related_name='platform_roles') + + class Meta: + db_table = 'platform_role_member' + constraints = [models.UniqueConstraint(fields=['role', 'user'], name='unique_platform_role_member')] + + +class PlatformGrant(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + subject_type = models.CharField(max_length=8, choices=[('ALL', '所有用户'), ('ROLE', '角色'), ('USER', '用户')]) + subject_id = models.CharField(max_length=64, blank=True, default='') + workspace_id = models.CharField(max_length=128, default='default', db_index=True) + resource_type = models.CharField(max_length=32) + target = models.CharField(max_length=128) + actions = models.JSONField(default=list) + expires_at = models.DateTimeField(null=True, blank=True) + granted_by = models.ForeignKey('users.User', null=True, on_delete=models.SET_NULL, related_name='+') + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'platform_grant' + constraints = [models.UniqueConstraint(fields=['workspace_id', 'subject_type', 'subject_id', 'resource_type', 'target'], name='unique_platform_grant')] + indexes = [models.Index(fields=['subject_type', 'subject_id'], name='platform_grant_subject_idx')] + + +class EffectiveResourcePermission(models.Model): + """Read-only SQL view merging legacy, role, user and public resource grants.""" + id = models.UUIDField(primary_key=True) + workspace_id = models.CharField(max_length=128) + user_id = models.UUIDField() + auth_target_type = models.CharField(max_length=128) + target = models.CharField(max_length=128) + auth_type = models.CharField(max_length=64) + permission_list = ArrayField(models.CharField(max_length=256)) + create_time = models.DateTimeField() + update_time = models.DateTimeField() + + class Meta: + managed = False + db_table = 'effective_resource_permission' diff --git a/apps/system_manage/parsers/__init__.py b/apps/system_manage/parsers/__init__.py new file mode 100644 index 000000000..2c6039867 --- /dev/null +++ b/apps/system_manage/parsers/__init__.py @@ -0,0 +1,38 @@ +"""Stable application boundary for independently deployed parser engines.""" +from pathlib import Path + +MINERU_INPUTS = frozenset({'pdf', 'png', 'jpg', 'jpeg', 'webp', 'bmp', 'tiff', 'gif', 'jp2'}) +NATIVE_INPUTS = frozenset({'docx', 'doc', 'xls', 'xlsx', 'csv', 'html', 'htm', 'md', 'txt'}) + + +def extension(name): + return Path(name).suffix.lower().lstrip('.') + + +def parse(parser, name, content, options, headers, progress_callback=None): + from rest_framework.exceptions import ValidationError + kind = extension(name) + if kind in NATIVE_INPUTS: + from .native import parse as native_parse + if progress_callback: + progress_callback({'stage': '使用内置处理器解析文档', 'percent': None}) + return native_parse(name, content) + protocol = options.get('protocol', 'legacy') + if protocol == 'legacy': + if parser.endpoint.rstrip('/').endswith('/v1'): + raise ValidationError('原版接口地址不能以 /v1 结尾,请修改解析服务配置') + if kind not in MINERU_INPUTS: + raise ValidationError('当前解析服务不支持该文件格式') + from .legacy import parse as engine_parse + elif protocol == 'v1': + from system_manage.ai_hub_parser_v1 import parse as engine_parse + else: + raise ValidationError('不支持的解析接口类型') + result = engine_parse(parser, name, content, options, headers, progress_callback=progress_callback) + result.setdefault('source', {}) + result['source'] = {'engine': 'mineru', 'protocol': protocol, + 'configured_version': options.get('engine_version', ''), **result['source']} + return result + + +__all__ = ['parse', 'extension', 'MINERU_INPUTS', 'NATIVE_INPUTS'] diff --git a/apps/system_manage/parsers/catalog.py b/apps/system_manage/parsers/catalog.py new file mode 100644 index 000000000..fc4580e94 --- /dev/null +++ b/apps/system_manage/parsers/catalog.py @@ -0,0 +1,53 @@ +"""Shared parser capability catalogue; installed services remain explicit resources.""" +BACKENDS_2 = { + 'pipeline': '传统管道', 'hybrid-auto-engine': '混合本地引擎', 'vlm-auto-engine': 'VLM 本地引擎', + 'hybrid-http-client': '混合远程推理', 'vlm-http-client': 'VLM 远程推理', +} +BACKENDS_3 = { + 'pipeline': '传统管道', 'hybrid-engine': '混合本地引擎', 'vlm-engine': 'VLM 本地引擎', + 'hybrid-http-client': '混合远程推理', 'vlm-http-client': 'VLM 远程推理', +} +TIERS = {'flash': 'Flash 快速', 'basic': 'Basic 基础', 'standard': 'Standard 标准', 'advanced': 'Advanced 高精度'} + + +def catalogue(): + return {'versions': [{'id': '2.7.4', 'protocol': 'legacy', 'backends': BACKENDS_2}, + {'id': '3.4.5', 'protocol': 'legacy', 'backends': BACKENDS_3}, + {'id': '4.x', 'protocol': 'v1', 'backends': TIERS}], + 'builtin': {'id': 'builtin', 'name': 'MaxKB 内置解析', 'backend': 'builtin', + 'formats': ['PDF', 'DOCX', 'DOC', 'XLSX', 'XLS', 'CSV', 'HTML', 'MD', 'TXT']}} + + +def validate_options(options, endpoint): + from urllib.parse import urlsplit + from rest_framework.exceptions import ValidationError + if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier', 'engine_version', 'server_url', 'effort'}: + raise ValidationError('解析参数无效') + version = options.get('engine_version', '4.x' if options.get('protocol') == 'v1' else '2.7.4') + spec = next((v for v in catalogue()['versions'] if v['id'] == version), None) + if not spec or options.get('protocol', spec['protocol']) != spec['protocol']: + raise ValidationError('解析版本与接口不匹配') + if spec['protocol'] == 'legacy': + if urlsplit(endpoint).path.endswith('/v1'): + raise ValidationError('2.x / 3.x 请填写解析 API 地址,不含 /v1') + backend = options.get('backend', 'pipeline') + if not isinstance(backend, str): + raise ValidationError('解析后端无效') + aliases = {'vlm-auto-engine': 'vlm-engine', 'hybrid-auto-engine': 'hybrid-engine'} if version == '3.4.5' else {} + if aliases.get(backend, backend) not in spec['backends']: + raise ValidationError('该版本不支持此解析后端') + if backend.endswith('-http-client'): + if not isinstance(options.get('server_url'), str): + raise ValidationError('请填写有效的远程推理地址') + remote = urlsplit(options.get('server_url', '')) + if remote.scheme not in ('http', 'https') or not remote.hostname or remote.username or remote.password or remote.query or remote.fragment: + raise ValidationError('请填写有效的远程推理地址') + elif options.get('tier', 'standard') not in TIERS: + raise ValidationError('解析档位无效') + if options.get('effort', 'medium') not in ('medium', 'high'): + raise ValidationError('解析精度无效') + if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'): + raise ValidationError('识别语言无效') + if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options): + raise ValidationError('解析参数无效') + return {**options, 'engine_version': version, 'protocol': spec['protocol']} diff --git a/apps/system_manage/parsers/legacy.py b/apps/system_manage/parsers/legacy.py new file mode 100644 index 000000000..e03a2c8ca --- /dev/null +++ b/apps/system_manage/parsers/legacy.py @@ -0,0 +1,72 @@ +"""Official MinerU 2.x sync HTTP, executed inside the Hub's durable background task.""" +import json +import re +import time +import requests +from rest_framework.exceptions import ValidationError + + +def parse(parser, name, content, options, headers, progress_callback=None): + from system_manage.ai_hub import text, MAX_INLINE_TEXT + if progress_callback: + progress_callback({'stage': 'MinerU 正在解析文档', 'percent': None}) + backend = options.get('backend', 'pipeline') + if options.get('engine_version') == '3.4.5': + backend = {'hybrid-auto-engine': 'hybrid-engine', 'vlm-auto-engine': 'vlm-engine'}.get(backend, backend) + fields = {'backend': backend, 'lang_list': options.get('language', 'ch'), + 'parse_method': 'ocr' if options.get('ocr') else 'auto', 'return_md': 'true', + 'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true', + 'formula_enable': str(options.get('formula', True)).lower(), + 'table_enable': str(options.get('table', True)).lower()} + if backend.endswith('-http-client'): + fields['server_url'] = options['server_url'] + if options.get('engine_version') == '3.4.5': + fields['effort'] = options.get('effort', 'medium') + try: + response = requests.post(parser.endpoint.rstrip('/') + '/file_parse', files={'files': (name, content)}, headers=headers, + data=fields, timeout=(10, 300), allow_redirects=False) + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + if response.status_code == 202 and options.get('engine_version') == '3.4.5': + try: + task_id = response.json().get('task_id') + if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id): + raise ValidationError('解析服务返回任务 ID 无效') + deadline = time.monotonic() + 300 + while response.status_code == 202: + if time.monotonic() >= deadline: + raise ValidationError('解析等待超时,后台任务 ID:' + task_id) + if progress_callback: + progress_callback({'stage': '等待远程解析完成', 'percent': None}) + time.sleep(2) + response = requests.get(parser.endpoint.rstrip('/') + '/tasks/' + task_id + '/result', + headers=headers, timeout=(10, 30), allow_redirects=False) + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + except ValueError as exc: + raise ValidationError('解析任务响应无效') from exc + if response.status_code != 200: + raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + try: + payload = response.json() + except ValueError as exc: + raise ValidationError('解析服务返回格式无效') from exc + entries = payload.get('results') if isinstance(payload, dict) else None + if not isinstance(entries, dict) or len(entries) != 1: + raise ValidationError('解析结果格式无效') + result = next(iter(entries.values())) + if not isinstance(result, dict): + raise ValidationError('解析结果格式无效') + middle = result.get('middle_json') or {} + if isinstance(middle, str): + try: + middle = json.loads(middle) + except ValueError as exc: + raise ValidationError('解析结构数据无效') from exc + if not isinstance(middle, dict): + raise ValidationError('解析结构数据无效') + if progress_callback: + progress_callback({'stage': '整理解析结果和图片', 'percent': None}) + return {'markdown': text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT), + 'source': {'version': payload.get('version', ''), 'backend': payload.get('backend', ''), 'middle_json': middle}, + 'parser_id': str(parser.id), 'images': result.get('images', {})} diff --git a/apps/system_manage/parsers/native.py b/apps/system_manage/parsers/native.py new file mode 100644 index 000000000..7058d6b85 --- /dev/null +++ b/apps/system_manage/parsers/native.py @@ -0,0 +1,71 @@ +"""Reuse MaxKB's built-in converters; MinerU receives only supported visual documents.""" +import base64 +import io +import mimetypes +from pathlib import Path +import shutil +import subprocess +import tempfile +import zipfile +from rest_framework.exceptions import ValidationError + + +def parse(name, content): + from django.core.files.uploadedfile import SimpleUploadedFile + from system_manage.ai_hub import text + from . import extension + suffix = extension(name) + if suffix == 'doc': + command = shutil.which('libreoffice') or shutil.which('soffice') + if command is None: + raise ValidationError('DOC 格式需要 LibreOffice,请先另存为 DOCX') + with tempfile.TemporaryDirectory(prefix='maxkb-office-') as directory: + root = Path(directory); source = root / 'input.doc'; source.write_bytes(content) + try: + result = subprocess.run([command, '-env:UserInstallation=' + (root / 'profile').as_uri(), + '--headless', '--convert-to', 'docx', '--outdir', directory, str(source)], + capture_output=True, timeout=120) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ValidationError('DOC 转换失败,请另存为 DOCX') from exc + output = root / 'input.docx' + if result.returncode or not output.is_file(): + raise ValidationError('DOC 转换失败,请另存为 DOCX') + parsed = parse(Path(name).stem + '.docx', output.read_bytes()) + parsed['source']['converted_from'] = 'doc' + return parsed + if suffix in ('docx', 'xlsx'): + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + if sum(item.file_size for item in archive.infolist()) > 200 * 1024 * 1024: + raise ValidationError('文档解压后超过大小限制') + except zipfile.BadZipFile as exc: + raise ValidationError('文件格式无效') from exc + converters = { + 'pdf': ('pdf_split_handle', 'PdfSplitHandle'), + 'docx': ('doc_split_handle', 'DocSplitHandle'), 'xlsx': ('xlsx_split_handle', 'XlsxSplitHandle'), + 'xls': ('xls_split_handle', 'XlsSplitHandle'), 'csv': ('csv_split_handle', 'CsvSplitHandle'), + 'html': ('html_split_handle', 'HTMLSplitHandle'), 'htm': ('html_split_handle', 'HTMLSplitHandle'), + 'md': ('text_split_handle', 'TextSplitHandle'), 'txt': ('text_split_handle', 'TextSplitHandle'), + } + if suffix not in converters: + raise ValidationError('内置处理器不支持该文件格式') + import importlib + module, converter = converters[suffix] + handler = getattr(importlib.import_module('common.handle.impl.text.' + module), converter)() + images = {} + def capture(files): + for file in files or []: + raw = file.meta.get('content') + if isinstance(raw, bytes): + mime = mimetypes.guess_type(file.file_name)[0] or 'image/jpeg' + images['./oss/file/' + str(file.id)] = f'data:{mime};base64,' + base64.b64encode(raw).decode() + upload = SimpleUploadedFile(Path(name.replace('\\', '/')).name, content) + result = handler.handle(upload, [], False, 2_000_000, lambda _: content, capture, 0) + sections = result if isinstance(result, list) else [result] + markdown = '\n\n'.join('\n\n'.join( + ((('# ' + item.get('title', '') + '\n') if item.get('title') else '') + item.get('content', '')) + for item in section.get('content', [])) for section in sections if isinstance(section, dict)) + for path, uri in images.items(): + markdown = markdown.replace(path, uri) + return {'markdown': text(markdown, '解析结果'), 'images': {}, + 'source': {'engine': 'maxkb-native', 'format': suffix}} diff --git a/apps/system_manage/platform_access.py b/apps/system_manage/platform_access.py new file mode 100644 index 000000000..a20c7c4f9 --- /dev/null +++ b/apps/system_manage/platform_access.py @@ -0,0 +1,350 @@ +"""Additive authorization: public + enabled roles + individual grants. + +The legacy permission table remains an independent source. New grants are never +materialized into it, so revocation cannot erase an owner's existing access. +""" +from django.core.exceptions import ValidationError as DjangoValidationError +import re +from django.db.models import Q +from django.db import transaction +from django.utils import timezone +from rest_framework.exceptions import PermissionDenied, ValidationError +from system_manage.models.platform_access import PlatformRole, PlatformRoleMember, PlatformGrant + +MODULES = {'application': 'AI应用', 'mindmap': '思维导图', 'knowledge': '知识库', 'tool': '工具管理', 'model': '模型管理'} +FEATURES = {'document_parse': '文档解析'} +ACTIONS = {'VIEW': '查看', 'USE': '使用', 'EDIT': '编辑', 'AUTHORIZE': '授权'} +DEFAULT_ROLE_ID = '00000000-0000-0000-0000-000000000001' +ADMIN_ROLE_ID = '00000000-0000-0000-0000-000000000002' +BUILTIN_ADMIN_USER_ID = 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab' + + +def is_admin(user): + return PlatformRoleMember.objects.filter(user_id=user.id, role_id=ADMIN_ROLE_ID, role__enabled=True).exists() + + +@transaction.atomic +def reconcile_builtin_roles(): + """One-time upgrade: import existing administrators without changing grants. + + Safe to repeat; only the initial creation imports legacy account types. + After that memberships are authoritative, including after demotion. + """ + from users.models import User + PlatformRole.objects.get_or_create(id=DEFAULT_ROLE_ID, defaults={ + 'name': '普通用户', 'description': '新激活用户自动关联', 'is_default': True, + 'modules': ['knowledge', 'mindmap']}) + admin_role, created = PlatformRole.objects.get_or_create(id=ADMIN_ROLE_ID, defaults={ + 'name': '系统管理员', 'description': '拥有全部平台管理权限', 'modules': list(MODULES)}) + if created: + PlatformRoleMember.objects.bulk_create([ + PlatformRoleMember(role=admin_role, user_id=pk) + for pk in User.objects.filter(role='ADMIN').values_list('id', flat=True) + ], ignore_conflicts=True) + return {'admin_role_created': created, + 'admin_members': PlatformRoleMember.objects.filter(role=admin_role).count()} + + +def _sync_legacy_role(user, operator_id=None): + """Keep old integrations as projections of the unified memberships.""" + from users.models import UserRoleAssignment + from users.serializers.user_manage_ext import invalidate_user_auth_cache + from users.serializers.user import update_user_role + role = 'ADMIN' if is_admin(user) else 'USER' + user.role = role + user.save(update_fields=['role', 'update_time']) + UserRoleAssignment.objects.filter(user_id=user.id, workspace_id='default', role_id__in=['ADMIN', 'USER']).delete() + UserRoleAssignment.objects.create(user_id=user.id, role_id=role, workspace_id='default', granted_by_id=operator_id) + if str(user.id) != BUILTIN_ADMIN_USER_ID: + update_user_role({'role_setting': [{'role_id': role, 'workspace_ids': ['None' if role == 'ADMIN' else 'default']}]}, user, operator_id) + # Clear immediately for callers in this transaction and again after commit + # to avoid another request repopulating caches with the previous state. + invalidate_user_auth_cache(user.id) + transaction.on_commit(lambda: invalidate_user_auth_cache(user.id)) + + +@transaction.atomic +def assign_roles(user_id, role_ids, operator_id=None): + from users.models import User + # All membership writers lock this row first, including bulk role editing. + PlatformRole.objects.select_for_update().get(pk=ADMIN_ROLE_ID) + user = User.objects.select_for_update().get(pk=user_id) + ids = {str(pk) for pk in role_ids} | {DEFAULT_ROLE_ID} + roles = list(PlatformRole.objects.select_for_update().filter(pk__in=ids).order_by('id')) + if len(roles) != len(ids): + raise ValidationError('角色不存在') + if str(user.id) == BUILTIN_ADMIN_USER_ID and ADMIN_ROLE_ID not in ids: + raise ValidationError('不能移除内置管理员角色') + if is_admin(user) and ADMIN_ROLE_ID not in ids and not PlatformRoleMember.objects.filter( + role_id=ADMIN_ROLE_ID, user__is_active=True).exclude(user_id=user.id).exists(): + raise ValidationError('至少保留一名启用的系统管理员') + PlatformRoleMember.objects.filter(user=user).exclude(role_id__in=ids).delete() + existing = {str(pk) for pk in user.platform_roles.values_list('role_id', flat=True)} + PlatformRoleMember.objects.bulk_create([PlatformRoleMember(user=user, role_id=pk) for pk in ids - existing]) + _sync_legacy_role(user, operator_id) + return user + + +def assign_legacy_role(user, role_id, operator_id=None): + """Adapter for registration and older clients; preserves custom roles.""" + if role_id not in ('ADMIN', 'USER'): + raise ValidationError('角色不合法') + ids = {str(pk) for pk in user.platform_roles.values_list('role_id', flat=True)} + ids.discard(ADMIN_ROLE_ID) + if role_id == 'ADMIN': + ids.add(ADMIN_ROLE_ID) + return assign_roles(user.id, ids, operator_id) + + +def require_admin(user): + if not is_admin(user): + raise PermissionDenied('仅管理员可管理角色和授权') + + +def roles_for(user): + return PlatformRole.objects.filter(enabled=True).filter(Q(is_default=True) | Q(members__user_id=user.id)).distinct() + + +def grants_for(user): + if not user.is_active: + return PlatformGrant.objects.none() + role_ids = [str(pk) for pk in roles_for(user).values_list('id', flat=True)] + return PlatformGrant.objects.filter(workspace_id='default').exclude(resource_type='FEATURE').filter( + Q(expires_at__isnull=True) | Q(expires_at__gt=timezone.now()) + ).filter(Q(subject_type='ALL') | Q(subject_type='USER', subject_id=str(user.id)) | + Q(subject_type='ROLE', subject_id__in=role_ids)) + + +def expand_actions(actions): + result = set(actions) + if 'EDIT' in result: + result |= {'VIEW', 'USE'} + if 'USE' in result or 'AUTHORIZE' in result: + result.add('VIEW') + return result + + +def resource_actions(user, resource_type, target, workspace='default'): + if not user.is_active or workspace != 'default': + return set() + if is_admin(user): + return set(ACTIONS) + actions = set() + for grant in grants_for(user).filter(resource_type=resource_type, target=str(target)): + actions.update(grant.actions) + return expand_actions(actions) + + +def module_list(user): + from system_manage.models import UserMenuSetting + if is_admin(user): + return list(MODULES) + result = set() + for role in roles_for(user): + result.update(role.modules) + # Preserve explicit legacy user menus; no implicit fallback grants once roles exist. + setting = UserMenuSetting.objects.filter(user_id=user.id).first() + if setting: + result.update(setting.menu_list or []) + for grant in grants_for(user).filter(resource_type='MODULE'): + if 'VIEW' in expand_actions(grant.actions): + result.add(grant.target) + return [key for key in MODULES if key in result] + + +def check_feature_request(request, user): + """Backend counterpart to menu visibility for business write operations. + + Read endpoints remain available for cross-module resource selectors; their + existing resource permission checks still apply. + """ + if is_admin(user) or request.method in ('GET', 'HEAD', 'OPTIONS'): + return + match = re.search(r'/workspace/default/(knowledge|application|tool|model)(?:/|$)', request.path) + if match and match.group(1) not in module_list(user): + from common.exception.app_exception import AppUnauthorizedFailed + raise AppUnauthorizedFailed(403, '未授权使用该功能') + + +def permission_strings(user): + """Expand new grants to the original endpoint permission vocabulary. + + Do not emit bare resource markers: legacy endpoints treat those as full + management. AUTHORIZE is separate from EDIT. + """ + from common.constants.permission_constants import (PermissionConstants, Resource, + ResourcePermission, ResourcePermissionGroup, get_permission_list_by_resource_group) + result = set() + for grant in grants_for(user).exclude(resource_type='MODULE'): + if grant.resource_type not in ('KNOWLEDGE', 'APPLICATION', 'TOOL', 'MODEL'): + continue + actions = expand_actions(grant.actions) + scope = f':/WORKSPACE/default/{grant.resource_type}/{grant.target}' + for level in ('VIEW', 'MANAGE'): + if level == 'VIEW' and 'VIEW' not in actions: + continue + if level == 'MANAGE' and 'EDIT' not in actions: + continue + group = ResourcePermissionGroup(Resource(grant.resource_type), ResourcePermission(level)) + for permission in get_permission_list_by_resource_group(group): + name = str(permission) + # Member/grant management requires AUTHORIZE, even for editors. + if any(part in name for part in ('WORKSPACE_USER_RESOURCE_PERMISSION', '+AUTH', '+ADD_MEMBER', '+REMOVE_MEMBER')): + continue + result.add(name + scope) + if 'AUTHORIZE' in actions: + for permission in PermissionConstants: + name = str(permission.value) + if name.startswith(grant.resource_type + '_WORKSPACE_USER_RESOURCE_PERMISSION:') or name.startswith(grant.resource_type + ':READ+AUTH'): + result.add(name + scope) + return result + + +def resource_models(): + from application.models import Application + from knowledge.models import Knowledge + from tools.models import Tool + from models_provider.models import Model + from system_manage.models.ai_hub import AIParserResource, AIService + return {'KNOWLEDGE': Knowledge, 'APPLICATION': Application, 'TOOL': Tool, + 'MODEL': Model, 'PARSER': AIParserResource, 'SERVICE': AIService} + + +def catalog(): + resources = [] + for kind, model in resource_models().items(): + fields = ['id', 'name'] + (['provider', 'model_type'] if kind == 'MODEL' else []) + for item in model.objects.filter(workspace_id='default').values(*fields).order_by('name'): + resources.append({**item, 'id': str(item['id']), 'resource_type': kind}) + return resources + + +def validate_target(resource_type, target): + if resource_type == 'MODULE': + if target not in MODULES: + raise ValidationError('功能无效') + elif resource_type == 'FEATURE': + if target not in FEATURES: + raise ValidationError('功能无效') + else: + model = resource_models().get(resource_type) + try: + valid = model is not None and model.objects.filter(pk=target, workspace_id='default').exists() + except (ValueError, TypeError, DjangoValidationError): + valid = False + if not valid: + raise ValidationError('资源不存在') + + +def effective_detail(user): + """Explain every source; legacy entries remain identifiable and revocable.""" + from system_manage.models import WorkspaceUserResourcePermission, UserMenuSetting + role_names = {str(r.id): r.name for r in roles_for(user)} + rows = [] + for grant in grants_for(user): + rows.append({'resource_type': grant.resource_type, 'target': grant.target, + 'actions': sorted(expand_actions(grant.actions)), + 'source': '所有用户' if grant.subject_type == 'ALL' else ( + role_names.get(grant.subject_id, '角色') if grant.subject_type == 'ROLE' else '单独授权'), + 'expires_at': grant.expires_at}) + for role in roles_for(user): + rows.extend({'resource_type': 'MODULE', 'target': module, 'actions': ['VIEW'], 'source': role.name} + for module in role.modules) + legacy_menu = UserMenuSetting.objects.filter(user_id=user.id).first() + if legacy_menu: + rows.extend({'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'source': '原用户菜单授权'} for m in legacy_menu.menu_list) + for grant in WorkspaceUserResourcePermission.objects.filter(user_id=user.id, workspace_id='default'): + actions = ['VIEW', 'USE'] if 'VIEW' in grant.permission_list else [] + if 'MANAGE' in grant.permission_list or 'ROLE' in grant.permission_list: + actions = list(ACTIONS) + rows.append({'resource_type': grant.auth_target_type, 'target': grant.target, 'actions': actions, 'source': '原用户资源授权'}) + for row in authorization_rows(): + if row.get('readonly') and ((row['subject_type'] == 'ROLE' and row['subject_id'] in role_names) or (row['subject_type'] == 'USER' and row['subject_id'] == str(user.id))): + rows.append({**row, 'source': role_names.get(row['subject_id'], '原用户菜单授权') + '(菜单附带)'}) + for kind in ('PARSER', 'SERVICE'): + model = resource_models()[kind] + for obj in model.objects.filter(workspace_id='default').filter(Q(owner_id=user.id) | Q(allowed_users__contains=[str(user.id)])): + owner = str(obj.owner_id) == str(user.id) + rows.append({'resource_type': kind, 'target': str(obj.id), 'actions': list(ACTIONS) if owner else ['VIEW', 'USE'], + 'source': '资源所有者' if owner else '原用户资源授权'}) + return {'modules': module_list(user), 'roles': [{'id': k, 'name': v} for k, v in role_names.items()], + 'permissions': rows, 'is_admin': is_admin(user)} + + +def authorization_rows(): + """Expose compatibility grants in the same management list, without migration.""" + from system_manage.models import UserMenuSetting, WorkspaceUserResourcePermission + rows = list(PlatformGrant.objects.values('id', 'subject_type', 'subject_id', 'resource_type', 'target', 'actions', 'expires_at')) + for role in PlatformRole.objects.exclude(id=ADMIN_ROLE_ID): + rows.extend({'id': f'role-menu:{role.id}:{m}', 'subject_type': 'ROLE', 'subject_id': str(role.id), + 'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'expires_at': None, + 'origin': '角色菜单'} for m in role.modules) + for setting in UserMenuSetting.objects.all(): + rows.extend({'id': f'user-menu:{setting.user_id}:{m}', 'subject_type': 'USER', 'subject_id': str(setting.user_id), + 'resource_type': 'MODULE', 'target': m, 'actions': ['VIEW'], 'expires_at': None, + 'origin': '原菜单授权'} for m in setting.menu_list or []) + for g in WorkspaceUserResourcePermission.objects.filter(workspace_id='default'): + # Owner/framework ROLE rows are not discretionary grants. + if 'ROLE' in g.permission_list: + continue + actions = list(ACTIONS) if 'MANAGE' in g.permission_list else (['VIEW', 'USE'] if 'VIEW' in g.permission_list else []) + rows.append({'id': f'legacy-resource:{g.id}', 'subject_type': 'USER', 'subject_id': str(g.user_id), + 'resource_type': g.auth_target_type, 'target': g.target, 'actions': actions, + 'expires_at': None, 'origin': '原资源授权'}) + for kind in ('PARSER', 'SERVICE'): + for obj in resource_models()[kind].objects.filter(workspace_id='default'): + rows.extend({'id': f'legacy-share:{kind}:{obj.id}:{uid}', 'subject_type': 'USER', 'subject_id': uid, + 'resource_type': kind, 'target': str(obj.id), 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '原资源授权'} for uid in obj.allowed_users) + for role in PlatformRole.objects.exclude(id=ADMIN_ROLE_ID): + if {'tool', 'mindmap'} & set(role.modules): + rows.append({'id': f'inherited-feature:{role.id}', 'subject_type': 'ROLE', 'subject_id': str(role.id), + 'resource_type': 'FEATURE', 'target': 'document_parse', 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '菜单附带', 'readonly': True}) + for setting in UserMenuSetting.objects.all(): + if {'tool', 'mindmap'} & set(setting.menu_list or []): + rows.append({'id': f'inherited-user-feature:{setting.user_id}', 'subject_type': 'USER', 'subject_id': str(setting.user_id), + 'resource_type': 'FEATURE', 'target': 'document_parse', 'actions': ['VIEW', 'USE'], + 'expires_at': None, 'origin': '菜单附带', 'readonly': True}) + return [row for row in rows if row['resource_type'] != 'FEATURE'] + + +def remove_compatibility_grant(key, expected=None): + from system_manage.models import UserMenuSetting, WorkspaceUserResourcePermission + row = next((r for r in authorization_rows() if str(r['id']) == key and r.get('origin')), None) + if row is None: + raise ValidationError('授权已变更,请刷新') + if row.get('readonly'): + raise ValidationError('此权限随原菜单生效,请调整菜单授权') + if expected and any(str(expected.get(k)) != str(row[k]) for k in ('subject_type', 'subject_id', 'resource_type', 'target')): + raise ValidationError('授权对象不匹配') + parts = key.split(':') + if parts[0] == 'role-menu': + role = PlatformRole.objects.select_for_update().get(pk=row['subject_id']) + role.modules = [m for m in role.modules if m != row['target']] + role.save(update_fields=['modules', 'updated_at']) + elif parts[0] == 'user-menu': + setting = UserMenuSetting.objects.select_for_update().get(user_id=row['subject_id']) + setting.menu_list = [m for m in setting.menu_list if m != row['target']] + setting.save(update_fields=['menu_list']) + elif parts[0] == 'legacy-resource': + WorkspaceUserResourcePermission.objects.filter(pk=parts[1]).delete() + from users.serializers.user_manage_ext import invalidate_user_auth_cache + transaction.on_commit(lambda: invalidate_user_auth_cache(row['subject_id'])) + elif parts[0] == 'legacy-share': + obj = resource_models()[row['resource_type']].objects.select_for_update().get(pk=row['target']) + obj.allowed_users = [uid for uid in obj.allowed_users if uid != row['subject_id']] + obj.save(update_fields=['allowed_users']) + + +def parser_feature_actions(user): + """Old tool/mindmap menus include parsing; new feature grants are explicit.""" + if not user.is_active: + return set() + if is_admin(user): + return {'VIEW', 'USE'} + from system_manage.models import UserMenuSetting + inherited = any({'tool', 'mindmap'} & set(role.modules) for role in roles_for(user)) + setting = UserMenuSetting.objects.filter(user_id=user.id).first() + inherited = inherited or bool(setting and {'tool', 'mindmap'} & set(setting.menu_list or [])) + return ({'VIEW', 'USE'} if inherited else set()) | resource_actions(user, 'FEATURE', 'document_parse') diff --git a/apps/system_manage/serializers/access_control_policy.py b/apps/system_manage/serializers/access_control_policy.py index e48f118a0..8c5b5b502 100644 --- a/apps/system_manage/serializers/access_control_policy.py +++ b/apps/system_manage/serializers/access_control_policy.py @@ -28,7 +28,7 @@ def _normalize_string_list(value): class AccessControlPolicySerializer(serializers.Serializer): - id = serializers.UUIDField(required=False) + id = serializers.UUIDField(read_only=True) name = serializers.CharField(required=True, max_length=128) description = serializers.CharField(required=False, allow_blank=True, default='') enabled = serializers.BooleanField(required=False, default=True) @@ -40,8 +40,8 @@ class AccessControlPolicySerializer(serializers.Serializer): allowed_regions = serializers.ListField(child=serializers.CharField(), required=False) effect_start_time = serializers.DateTimeField(required=False, allow_null=True) effect_end_time = serializers.DateTimeField(required=False, allow_null=True) - application_count = serializers.IntegerField(required=False) - last_applied_time = serializers.DateTimeField(required=False, allow_null=True) + application_count = serializers.IntegerField(read_only=True) + last_applied_time = serializers.DateTimeField(read_only=True) create_time = serializers.DateTimeField(required=False) update_time = serializers.DateTimeField(required=False) content = serializers.CharField(required=False) @@ -99,9 +99,14 @@ class AccessControlPolicySerializer(serializers.Serializer): return sorted(set(value or [])) def validate_allowed_device_types(self, value): + from system_manage.access_control import DEVICE_TYPES + if set(value) - set(DEVICE_TYPES): + raise AppApiException(500, '设备类型仅支持桌面端、移动终端、平板、服务器') return _normalize_string_list(value) def validate_allowed_regions(self, value): + if len(value) > 100 or any(not item.strip() or len(item.strip()) > 64 for item in value): + raise AppApiException(500, '区域名称须为1~64字,最多100项') return _normalize_string_list(value) def validate(self, attrs): @@ -113,8 +118,8 @@ class AccessControlPolicySerializer(serializers.Serializer): if access_time_enabled and (access_start_time is None or access_end_time is None): raise AppApiException(500, '启用访问时段限制后,必须设置开始和结束时间') - if access_start_time and access_end_time and access_start_time >= access_end_time: - raise AppApiException(500, '访问结束时间必须晚于开始时间') + if access_time_enabled and access_start_time == access_end_time: + raise AppApiException(500, '开始和结束时间不能相同') if effect_start_time and effect_end_time and effect_start_time >= effect_end_time: raise AppApiException(500, '生效结束时间必须晚于开始时间') return attrs @@ -151,28 +156,15 @@ class AccessControlPolicySerializer(serializers.Serializer): class AccessControlPolicyApplySerializer(serializers.Serializer): target_type = serializers.ChoiceField(choices=AccessControlPolicyApplicationRecord.TargetType.choices) - target_name = serializers.CharField(required=True, max_length=128) - target_identifier = serializers.CharField(required=False, allow_blank=True, default='', max_length=128) + target_name = serializers.CharField(required=False, allow_blank=True, default='', max_length=128) + target_identifier = serializers.CharField(required=True, max_length=128) remark = serializers.CharField(required=False, allow_blank=True, default='') @transaction.atomic def save(self, policy: AccessControlPolicy, operator): self.is_valid(raise_exception=True) - validated_data = self.validated_data - AccessControlPolicyApplicationRecord.objects.create( - policy=policy, - policy_name=policy.name, - target_type=validated_data.get('target_type'), - target_name=validated_data.get('target_name').strip(), - target_identifier=validated_data.get('target_identifier', '').strip(), - remark=validated_data.get('remark', '').strip(), - operator=operator, - operator_name=getattr(operator, 'nick_name', '') or getattr(operator, 'username', ''), - ) - policy.application_count = (policy.application_count or 0) + 1 - policy.last_applied_time = timezone.now() - policy.save(update_fields=['application_count', 'last_applied_time', 'update_time']) - return {'policy_id': str(policy.id), 'target_name': validated_data.get('target_name').strip()} + from system_manage.access_control import bind + return bind(policy.id, operator, self.validated_data) class AccessControlPolicyApplicationRecordSerializer(serializers.Serializer): diff --git a/apps/system_manage/serializers/menu_setting.py b/apps/system_manage/serializers/menu_setting.py index 2fac11776..7366a732a 100644 --- a/apps/system_manage/serializers/menu_setting.py +++ b/apps/system_manage/serializers/menu_setting.py @@ -133,6 +133,11 @@ class MenuSettingSerializer(serializers.Serializer): @staticmethod def get_current_menu_list(user): + from system_manage.platform_access import module_list + return module_list(user) + + @staticmethod + def get_legacy_menu_list(user): if MenuSettingSerializer._is_system_admin(user): return MenuSettingSerializer._get_admin_menu_list() user_id = getattr(user, 'id', user) diff --git a/apps/system_manage/serializers/system.py b/apps/system_manage/serializers/system.py index fe507e34e..c42c0330a 100644 --- a/apps/system_manage/serializers/system.py +++ b/apps/system_manage/serializers/system.py @@ -48,9 +48,5 @@ class SystemProfileSerializer(serializers.Serializer): class SystemConfigSerializer(serializers.Serializer): @staticmethod def get_config(): - # Prefer the bundled workbench: it uses the current login through its parent. - bundled = os.path.join(settings.STATIC_ROOT, 'admin', 'mindmap-workbench', 'index.html') - return { - 'mindmap_url': (CONFIG.get_admin_path() + '/mindmap-workbench/index.html') - if os.path.isfile(bundled) else CONFIG.get('MINDMAP_URL', '') - } + # Kept for older clients; the workbench is now a native MaxKB route. + return {'mindmap_url': CONFIG.get_admin_path() + '/mindmap'} diff --git a/apps/system_manage/serializers/user_resource_permission.py b/apps/system_manage/serializers/user_resource_permission.py index 9ce375ec3..e51251a9b 100644 --- a/apps/system_manage/serializers/user_resource_permission.py +++ b/apps/system_manage/serializers/user_resource_permission.py @@ -157,6 +157,9 @@ class UserResourcePermissionSerializer(serializers.Serializer): workspace_manage = is_workspace_manage(user_id, workspace_id) if workspace_manage: return True + from system_manage.platform_access import resource_actions + if 'USE' in resource_actions(User.objects.get(pk=user_id), auth_target_type, resource_id, workspace_id): + return True wurp = QuerySet(WorkspaceUserResourcePermission).filter(auth_target_type=auth_target_type, workspace_id=workspace_id, user=user_id, target=resource_id).first() diff --git a/apps/system_manage/test_access_control.py b/apps/system_manage/test_access_control.py new file mode 100644 index 000000000..53cd3a2c4 --- /dev/null +++ b/apps/system_manage/test_access_control.py @@ -0,0 +1,234 @@ +from datetime import datetime, time, timedelta, timezone as tz +from unittest import TestCase +from unittest.mock import patch +from types import SimpleNamespace +from django.db import transaction +from django.test import RequestFactory +from rest_framework.exceptions import ValidationError +from common.exception.app_exception import AppUnauthorizedFailed, AppApiException +from system_manage import access_control as ac +from system_manage.models import AccessControlPolicy, AccessControlPolicyApplicationRecord, Log +from system_manage.serializers.access_control_policy import AccessControlPolicySerializer +from users.models import User + + +class AccessTests(TestCase): + def setUp(self): + self.atomic = transaction.atomic() + self.atomic.__enter__() + self.user = User.objects.create(username='CODEX_ACCESS_TEST', nick_name='测试管理员', role='ADMIN') + self.policy = AccessControlPolicy.objects.create(name='CODEX_ACCESS_TEST') + self.now = datetime(2026, 10, 26, 2, 0, tzinfo=tz.utc) + + def tearDown(self): + transaction.set_rollback(True) + self.atomic.__exit__(None, None, None) + + def request(self, ip='10.1.2.3', agent='Mozilla/5.0 (Windows NT 10.0)', **meta): + return RequestFactory().get('/chat/api/open', REMOTE_ADDR=ip, HTTP_USER_AGENT=agent, **meta) + + def bind(self, target='application:sample'): + with patch.object(ac, 'targets', return_value=[{'id': target, 'label': '测试服务'}]): + return ac.bind(self.policy.id, self.user, {'target_type': 'SERVICE', 'target_identifier': target}) + + def test_no_binding_does_not_restrict_or_audit(self): + ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.count(), 0) + + def test_existing_history_does_not_activate(self): + AccessControlPolicyApplicationRecord.objects.create(policy=self.policy, policy_name=self.policy.name, + target_type='SERVICE', target_identifier='application:sample', target_name='历史记录') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.count(), 0) + + def test_service_binding_and_deduplicated_audit(self): + self.bind(); request = self.request() + ac.enforce(request, 'application', 'sample'); ac.enforce(request, 'application', 'sample') + self.assertEqual(Log.objects.count(), 1) + self.assertEqual(Log.objects.first().status, 200) + + def test_device_denied(self): + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + self.assertEqual(Log.objects.first().status, 403) + self.assertNotIn('body', Log.objects.first().details) + + def test_update_applies_to_next_request(self): + self.bind(); ac.enforce(self.request(), 'application', 'sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_disabled_policy(self): + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.enabled = False; self.policy.save() + ac.enforce(self.request(), 'application', 'sample') + + def test_unbind_keeps_history(self): + item = self.bind(); ac.unbind(self.policy.id, self.user, item) + self.assertEqual(ac.configuration()[1]['bindings'], []) + self.assertEqual(AccessControlPolicyApplicationRecord.objects.count(), 2) + + def test_duplicate_binding(self): + self.bind() + with self.assertRaises(ValidationError): self.bind() + + def test_nonexistent_service(self): + with patch.object(ac, 'targets', return_value=[]), self.assertRaises(ValidationError): + ac.bind(self.policy.id, self.user, {'target_type': 'SERVICE', 'target_identifier': 'application:missing'}) + + def test_device_cidr(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.2.3/24'}) + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'mindmap', 'sample') + ac.enforce(self.request(ip='10.2.2.3'), 'mindmap', 'sample') + + def test_ipv6(self): + self.assertTrue(ac.contains('2001:db8::1', '2001:db8::/32')) + self.assertFalse(ac.contains('10.1.1.1', '2001:db8::/32')) + + def test_invalid_network(self): + for value in ['invalid', None, '10.500.0.1']: + with self.subTest(value=value), self.assertRaises(ValidationError): ac.network(value) + + def test_all_policies_must_allow(self): + self.bind() + other = AccessControlPolicy.objects.create(name='另一策略', allowed_device_types=['移动终端']) + row, config = ac.configuration() + config['bindings'].append({**config['bindings'][0], 'policy_id': str(other.id)}) + row.meta = config; row.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_unknown_region(self): + self.bind(); self.policy.allowed_regions = ['北京']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_region_mapping_and_change(self): + self.bind(); self.policy.allowed_regions = ['北京']; self.policy.save() + ac.save_regions([{'name': '北京', 'network': '10.1.0.0/16'}]) + ac.enforce(self.request(), 'application', 'sample') + ac.save_regions([]) + with self.assertRaises(AppUnauthorizedFailed): ac.enforce(self.request(), 'application', 'sample') + + def test_overlapping_regions(self): + with self.assertRaises(ValidationError): + ac.save_regions([{'name': '北京', 'network': '10.0.0.0/8'}, {'name': '上海', 'network': '10.1.0.0/16'}]) + + def test_spoofed_forwarded_ignored(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': ''}): + self.assertEqual(ac.client_ip(self.request(HTTP_X_FORWARDED_FOR='8.8.8.8')), '10.1.2.3') + + def test_trusted_proxy_chain(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32,10.0.0.0/8'}): + self.assertEqual(ac.client_ip(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='1.1.1.1, 8.8.8.8, 10.2.2.2')), '8.8.8.8') + + def test_invalid_forwarded_fails_closed(self): + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32'}): + with self.assertRaises(AppUnauthorizedFailed): + ac.client_ip(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='invalid')) + + def test_invalid_proxy_address_cannot_bypass_device_policy(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.0.0/16'}) + with patch.dict('os.environ', {'MAXKB_ACCESS_TRUSTED_PROXIES': '127.0.0.1/32'}): + with self.assertRaises(AppUnauthorizedFailed): + ac.enforce(self.request(ip='127.0.0.1', HTTP_X_FORWARDED_FOR='invalid'), 'mindmap', 'sample') + self.assertEqual(Log.objects.first().details['reason'], '来源 IP 无效') + + def test_device_classification(self): + for agent, expected in [('Mozilla Windows', '桌面端'), ('iPhone Mobile', '移动终端'), + ('iPad', '平板'), ('Android Tablet', '平板'), ('curl/8', '服务器'), ('', '未知')]: + with self.subTest(agent=agent): self.assertEqual(ac.device_type(self.request(agent=agent)), expected) + + def test_time_boundaries(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(10); self.policy.access_end_time = time(11) + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + self.assertTrue(ac.policy_reason(self.policy, self.now + timedelta(hours=1), '桌面端', set())) + + def test_weekday(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(9); self.policy.access_end_time = time(18) + self.policy.allowed_weekdays = [2] + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + + def test_overnight_start_day(self): + self.policy.access_time_enabled = True; self.policy.access_start_time = time(22); self.policy.access_end_time = time(6) + self.policy.allowed_weekdays = [1] + for utc_hour in [15, 19]: + self.assertEqual(ac.policy_reason(self.policy, self.now.replace(hour=utc_hour), '桌面端', set()), '') + self.assertTrue(ac.policy_reason(self.policy, self.now.replace(hour=22), '桌面端', set())) + + def test_effective_boundaries(self): + self.policy.allowed_device_types = ['移动终端'] + self.policy.effect_start_time = self.now + timedelta(seconds=1) + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + self.policy.effect_start_time = self.now + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + self.policy.effect_end_time = self.now + self.assertEqual(ac.policy_reason(self.policy, self.now, '桌面端', set()), '') + + def test_invalid_time_denied(self): + self.policy.access_time_enabled = True + self.assertTrue(ac.policy_reason(self.policy, self.now, '桌面端', set())) + + def test_serializer_overnight_and_equal(self): + values = {'name': '夜间策略', 'access_time_enabled': True, 'access_start_time': '22:00', 'access_end_time': '06:00'} + self.assertTrue(AccessControlPolicySerializer(data=values).is_valid(raise_exception=True)) + values['access_end_time'] = '22:00' + with self.assertRaises(AppApiException): AccessControlPolicySerializer(data=values).is_valid(raise_exception=True) + + def test_serializer_device_options(self): + with self.assertRaises(AppApiException): + AccessControlPolicySerializer(data={'name': '无效设备策略', 'allowed_device_types': ['任意设备']}).is_valid(raise_exception=True) + + def test_auth_does_not_read_lazy_user(self): + self.bind() + class PendingRequest: + META = {'REMOTE_ADDR': '10.1.2.3', 'HTTP_USER_AGENT': 'Mozilla'} + path = '/chat/api/chat' + @property + def user(self): raise AssertionError('authentication recursion') + ac.enforce(PendingRequest(), 'application', 'sample') + + def test_token_auth_enforces_published_and_api_requests(self): + from common.auth.authenticate import TokenAuth + from common.constants.permission_constants import ChatAuth + auth = ChatAuth([], [], 'client', 'APPLICATION_API_KEY', 'sample') + handler = SimpleNamespace(support=lambda *args: True, handle=lambda *args: (None, auth)) + self.bind(); self.policy.allowed_device_types = ['移动终端']; self.policy.save() + request = self.request(); request.META['HTTP_AUTHORIZATION'] = 'Bearer test' + with patch('common.auth.authenticate.handles', [handler]), patch('traceback.print_stack'), self.assertRaises(AppUnauthorizedFailed): + TokenAuth().authenticate(request) + + def test_token_cannot_call_another_application(self): + from common.auth.authenticate import TokenAuth + from common.constants.permission_constants import ChatAuth + auth = ChatAuth([], [], 'client', 'APPLICATION_API_KEY', 'sample') + handler = SimpleNamespace(support=lambda *args: True, handle=lambda *args: (None, auth)) + request = self.request(); request.META['HTTP_AUTHORIZATION'] = 'Bearer test' + request.resolver_match = SimpleNamespace(kwargs={'application_id': 'different'}) + with patch('common.auth.authenticate.handles', [handler]), patch('traceback.print_stack'), self.assertRaises(AppUnauthorizedFailed): + TokenAuth().authenticate(request) + + def test_direct_workbench_cannot_bypass_device_binding(self): + ac.bind(self.policy.id, self.user, {'target_type': 'DEVICE', 'target_identifier': '10.1.0.0/16'}) + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce_task(self.request(), nodes=[{'type': 'render'}]) + + def test_retry_checks_original_service(self): + self.bind(target='mindmap:sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + with self.assertRaises(AppUnauthorizedFailed): ac.enforce_task(self.request(), 'sample', [{'type': 'render'}]) + + def test_parser_check_precedes_file_storage(self): + from unittest.mock import Mock + from django.core.files.uploadedfile import SimpleUploadedFile + from knowledge.serializers.document import DocumentSerializers + self.bind(target='parser:sample') + self.policy.allowed_device_types = ['移动终端']; self.policy.save() + serializer = DocumentSerializers.Split(data={}, context={'request': self.request()}) + serializer._validated_data = {} + serializer._errors = {} + serializer._data = {'workspace_id': 'default', 'knowledge_id': 'sample'} + serializer.is_valid = Mock(return_value=True) + with patch('knowledge.serializers.document.QuerySet') as query, patch('knowledge.models.File.save') as store: + query.return_value.filter.return_value.first.return_value = SimpleNamespace(meta={'parser_id': 'sample'}) + with self.assertRaises(AppUnauthorizedFailed): serializer.parse({'file': [SimpleUploadedFile('CODEX.pdf', b'test')]}) + store.assert_not_called() diff --git a/apps/system_manage/test_access_control_decisions.py b/apps/system_manage/test_access_control_decisions.py new file mode 100644 index 000000000..28cc56a4c --- /dev/null +++ b/apps/system_manage/test_access_control_decisions.py @@ -0,0 +1,72 @@ +from datetime import datetime, timedelta, timezone +from unittest import TestCase + +from django.db import transaction +from rest_framework.exceptions import ValidationError + +from system_manage.access_control_decisions import list_decisions +from system_manage.models import Log + + +class DecisionTests(TestCase): + def setUp(self): + self.atomic = transaction.atomic() + self.atomic.__enter__() + self.start = datetime(2026, 10, 27, 0, 0, tzinfo=timezone.utc) + for index in range(235): + denied = index % 2 == 0 + row = Log.objects.create(menu='接入控制', operate='拒绝访问' if denied else '允许访问', + status=403 if denied else 200, ip_address='192.0.2.1' if denied else '2001:db8::1', + operation_object={'target': 'CODEX_DECISION_TEST'}, + details={'device_type': '移动终端' if denied else '桌面端', + 'reason': '设备类型不允许' if denied else '', 'id': 'untrusted', 'status': 999}) + Log.objects.filter(pk=row.id).update(create_time=self.start + timedelta(minutes=index)) + Log.objects.create(menu='其他菜单', operate='拒绝访问', status=403, ip_address='192.0.2.1') + Log.objects.create(menu='接入控制', operate='更新区域网段', status=200, ip_address='192.0.2.1') + + def tearDown(self): + transaction.set_rollback(True) + self.atomic.__exit__(None, None, None) + + def test_pages_cover_beyond_old_limit(self): + pages = [list_decisions({'page': page, 'page_size': 100}) for page in (1, 2, 3)] + self.assertEqual([len(page['records']) for page in pages], [100, 100, 35]) + self.assertEqual({page['total'] for page in pages}, {235}) + self.assertEqual(len({row['id'] for page in pages for row in page['records']}), 235) + self.assertTrue(pages[0]['records'][0]['time'] > pages[1]['records'][0]['time']) + self.assertEqual(pages[0]['records'][0]['status'], 403) + + def test_combined_filters_and_timezone_boundaries(self): + page = list_decisions({'start_time': '2026-10-27T08:10:00', + 'end_time': '2026-10-27T08:20:00+08:00', 'result': 'denied', + 'ip': '192.0.2', 'device_type': '移动终端', 'reason': '设备'}) + self.assertEqual(page['total'], 6) + self.assertEqual(page['records'][0]['time'], self.start + timedelta(minutes=20)) + self.assertEqual(page['records'][-1]['time'], self.start + timedelta(minutes=10)) + + def test_filters_independently(self): + for params, total in [({'result': 'allowed'}, 117), ({'denied': 'true'}, 118), + ({'ip': 'db8::'}, 117), ({'device_type': '移动终端'}, 118), + ({'reason': '不允许'}, 118), ({'device_type': '未知'}, 0)]: + with self.subTest(params=params): + self.assertEqual(list_decisions(params)['total'], total) + + def test_empty_and_out_of_range_pages(self): + empty = list_decisions({'ip': 'absent', 'page': 99}) + self.assertEqual((empty['records'], empty['total'], empty['current']), ([], 0, 1)) + self.assertEqual(list_decisions({'page': 99, 'page_size': 100})['current'], 3) + + def test_reject_invalid_parameters(self): + for params in [{'page': 0}, {'page_size': 101}, {'page': 'abc'}, {'result': 'other'}, + {'device_type': 'other'}, {'start_time': 'invalid'}, + {'start_time': '2026-10-28T00:00:00', 'end_time': '2026-10-27T00:00:00'}]: + with self.subTest(params=params), self.assertRaises(ValidationError): + list_decisions(params) + + def test_stable_order_for_equal_times(self): + Log.objects.filter(menu='接入控制').update(create_time=self.start) + first = list_decisions({'page_size': 100}) + second = list_decisions({'page': 2, 'page_size': 100}) + ids = [row['id'] for row in first['records'] + second['records']] + self.assertEqual(ids, sorted(ids, reverse=True)) + self.assertEqual(len(set(ids)), 200) diff --git a/apps/system_manage/test_ai_hub.py b/apps/system_manage/test_ai_hub.py index 494ac8ed6..3d8be5770 100644 --- a/apps/system_manage/test_ai_hub.py +++ b/apps/system_manage/test_ai_hub.py @@ -102,7 +102,7 @@ class HubTests(TestCase): import requests from types import SimpleNamespace from system_manage.views.ai_hub import ParserTest - p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1'}) + p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}}) with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id) get.assert_not_called() get.return_value = Mock(status_code=302) @@ -149,7 +149,25 @@ class HubTests(TestCase): task.refresh_from_db() self.assertEqual(task.status, 'SUCCESS') self.assertEqual(task.result['original'], '# 内容\n数值42') - self.assertEqual(task.result['source']['pages'], [1]) + self.assertEqual(task.result['source']['middle_json']['pages'], [1]) + + def test_builtin_docx_works_without_external_parser(self): + import io + from docx import Document as WordDocument + document = WordDocument() + document.add_heading('Native42', 0) + document.add_paragraph('保留原文数值42') + buffer = io.BytesIO(); document.save(buffer) + task = hub.submit('default', self.user, {'nodes': [ + {'type': 'parse', 'processor': 'builtin'}, {'type': 'render'}]}, + SimpleUploadedFile('native.docx', buffer.getvalue())) + with patch('requests.post') as external: + execute(claim()) + external.assert_not_called() + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS', task.error) + self.assertIn('42', task.result['markdown']) + self.assertEqual(task.result['source']['engine'], 'maxkb-native') def test_parser_disabled_or_changed_after_submit_fails(self): parser = self.parser() @@ -171,6 +189,19 @@ class HubTests(TestCase): self.assertEqual(task.result['original'], '# 原稿') self.assertEqual(task.result['markdown'], '# 整理稿') + def test_model_params_form_defaults_win_over_absent_node_values(self): + from models_provider.models import Model + model = Model.objects.create(name='默认参数', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', + model_params_form=[{'field': 'temperature', 'default_value': 0.7}, {'field': 'max_tokens', 'default_value': 8192}]) + task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]}) + fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿') + with patch('models_provider.tools.get_model', return_value=fake) as get: + execute(claim()) + self.assertEqual(get.call_args.kwargs['temperature'], 0.7) + self.assertEqual(get.call_args.kwargs['max_tokens'], 8192) + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS') + def test_provider_exception_is_sanitized(self): parser = self.parser() task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf')) diff --git a/apps/system_manage/test_ai_hub_organize.py b/apps/system_manage/test_ai_hub_organize.py new file mode 100644 index 000000000..3da33f69c --- /dev/null +++ b/apps/system_manage/test_ai_hub_organize.py @@ -0,0 +1,44 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch +from system_manage.ai_hub_organize import organize, split_markdown, token_bound + + +class OrganizeTests(unittest.TestCase): + def test_split_preserves_long_unbroken_unicode_and_image_tokens(self): + text = '# 标题\n' + '超长段落42' * 8000 + '\nAIHUBIMAGE0001TOKEN\n结束' + chunks = split_markdown(text, 2048) + self.assertEqual(''.join(chunks), text) + self.assertTrue(all(token_bound(chunk) <= 2048 for chunk in chunks)) + self.assertTrue(any('AIHUBIMAGE0001TOKEN' in chunk for chunk in chunks)) + + def test_long_document_is_organized_and_merged_with_checkpoints(self): + source = '\n'.join(f'## section {i}\n' + 'fact42 ' * 1000 for i in range(12)) + stages, saved, calls = [], [], [] + def invoke(prompt, content): + calls.append(content) + return SimpleNamespace(content='# Result\n- fact42', response_metadata={}) + result = organize(source, '整理事实', invoke, context_tokens=8192, output_tokens=1024, + progress=stages.append, checkpoint=saved.append) + self.assertIn('fact42', result) + self.assertGreater(len(calls), 2) + self.assertGreater(len(saved), 2) + self.assertTrue(any('合并' in stage['stage'] for stage in stages)) + + def test_truncated_output_is_not_accepted_as_success(self): + with self.assertRaisesRegex(ValueError, '截断'): + organize('facts', '整理', lambda *args: SimpleNamespace( + content='partial', response_metadata={'finish_reason': 'length'})) + + def test_failed_chunk_keeps_completed_checkpoints_and_sanitizes_provider_error(self): + saved, calls = [], [] + def invoke(*args): + calls.append(1) + if len(calls) > 1: + raise RuntimeError('private credential must not escape') + return '# saved42' + with patch('system_manage.ai_hub_organize.time.sleep'): + with self.assertRaisesRegex(ValueError, '已保留') as failure: + organize('word42 ' * 5000, '整理', invoke, context_tokens=4096, output_tokens=512, checkpoint=saved.append) + self.assertNotIn('private', str(failure.exception)) + self.assertEqual(saved, [['# saved42']]) diff --git a/apps/system_manage/test_document_export.py b/apps/system_manage/test_document_export.py new file mode 100644 index 000000000..a69f88c2b --- /dev/null +++ b/apps/system_manage/test_document_export.py @@ -0,0 +1,89 @@ +"""No database or external model is needed for these export contract checks.""" +import base64 +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +from docx import Document +from PIL import Image +from system_manage.document_export import export_document, ExportError + + +class DocumentExportTests(unittest.TestCase): + def test_real_docx_preserves_structure_and_merged_cells(self): + image = io.BytesIO() + Image.new('RGB', (40, 20), 'blue').save(image, format='PNG') + html = '
加粗斜体2
' + html += '| 表头 | |
|---|---|
| 合并 | A |
| B | |
结束标记
' + raw, mime = export_document(html) + self.assertTrue(raw.startswith(b'PK')) + self.assertIn('wordprocessingml', mime) + doc = Document(io.BytesIO(raw)) + self.assertEqual(len(doc.inline_shapes), 1) + self.assertEqual(len(doc.tables[0].rows), 3) + self.assertEqual(doc.tables[0].cell(1, 0).text, doc.tables[0].cell(2, 0).text) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + xml = archive.read('word/document.xml').decode() + for expected in ('中文报告', '结束标记', 'w:numPr', 'w:hyperlink', 'w:tblHeader', 'w:vMerge', 'w:gridSpan', 'w:vertAlign'): + self.assertIn(expected, xml) + + def test_image_resolution_never_fetches_external_urls(self): + for src in ('https://example.com/private', 'file:///etc/passwd', 'data:image/png;base64,AAAA'): + with self.assertRaises(ExportError): + export_document('正文
', **kwargs) + + def test_long_table_grid_is_not_rebuilt_per_cell(self): + from docx.table import Table + read_grid = Table._cells.fget + calls = [] + def counted(table): + calls.append(1) + return read_grid(table) + html = '| 编号 | 内容 |
|---|---|
| {i} | 完整内容 {i} |
| x | ' * 31 + '
正文
', 'pdf') + + def test_conversion_uses_private_profile_and_removes_temporary_files(self): + directories = [] + def convert(args, **kwargs): + root = Path(args[args.index('--outdir') + 1]); directories.append(root) + self.assertTrue(any(arg.startswith('-env:UserInstallation=file:') for arg in args)) + self.assertEqual(kwargs['timeout'], 90) + self.assertNotIn('shell', kwargs) + (root / 'document.pdf').write_bytes(b'%PDF-1.7\ncontract-test') + return type('Result', (), {'returncode': 0})() + with patch('system_manage.document_export.shutil.which', return_value='soffice'), patch('system_manage.document_export.subprocess.run', side_effect=convert): + raw, mime = export_document('正文
', 'pdf') + self.assertEqual(mime, 'application/pdf') + self.assertTrue(raw.startswith(b'%PDF')) + self.assertTrue(all(not path.exists() for path in directories)) + + +if __name__ == '__main__': + unittest.main() diff --git a/apps/system_manage/urls.py b/apps/system_manage/urls.py index f3f2cef21..72cbea963 100644 --- a/apps/system_manage/urls.py +++ b/apps/system_manage/urls.py @@ -1,11 +1,20 @@ from django.urls import path from . import views -from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport +from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport, DocumentExport +from .views.platform_access import AccessView, RoleView, GrantsView, UserAccessView +from .views.access_control_policy import AccessControlRuntimeView, AccessControlDecisionView app_name = "system_manage" # @formatter:off urlpatterns = [ + path('platform_access', AccessView.as_view()), + path('platform_access/roles', RoleView.as_view()), + path('platform_access/roles/