diff --git a/apps/common/model_runtime.py b/apps/common/model_runtime.py new file mode 100644 index 000000000..6819dd162 --- /dev/null +++ b/apps/common/model_runtime.py @@ -0,0 +1,202 @@ +"""Single-owner, priority-ordered weight loading. No engine or framework dependency.""" +import contextlib +import itertools +import queue +import threading +import time +import uuid +from concurrent.futures import Future + + +def loading_policy(value=None): + value = {} if value is None else value + if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}: + raise ValueError('加载策略无效') + result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50), + 'idle_seconds': value.get('idle_seconds', 300)} + if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'): + raise ValueError('加载方式无效') + if type(result['priority']) is not int or not 0 <= result['priority'] <= 100: + raise ValueError('加载优先级须为0~100的整数') + if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400: + raise ValueError('空闲时间须为30~86400秒') + return result + + +class ModelRuntime: + def __init__(self, specs, adapter, clock=time.monotonic): + self.specs, self.adapter, self.clock = specs, adapter, clock + self.entries = {} + self.lock = threading.RLock() + self.queue = queue.PriorityQueue() + self.sequence = itertools.count() + self.load_sequence = itertools.count(1) + self.worker = None + + def _entry(self, model_id): + model_id = str(model_id) + spec = self.specs(model_id) + spec['policy'] = loading_policy(spec.get('policy')) + entry = self.entries.get(model_id) + if entry and entry['spec']['fingerprint'] != spec['fingerprint']: + if entry['leases'] or entry['state'] == 'LOADING': + raise ValueError('模型配置已变更,请等待当前请求结束') + self._discard(entry) + entry = None + if entry is None: + entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None, + 'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {}, + 'last_used': self.clock(), 'loaded_at': None, 'started_at': None, + 'load_order': None} + self.entries[model_id] = entry + previous = entry['spec']['policy'] + entry['spec'] = spec + if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']: + entry['generation'] += 1 + self._put(entry) + return entry + + def _put(self, entry): + self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation'])) + + def start(self): + with self.lock: + if self.worker is None: + self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True) + self.worker.start() + + def submit(self, model_id, start=True): + with self.lock: + entry = self._entry(model_id) + if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']): + entry.update(state='UNLOADED', value=None, info={}) + if entry['state'] == 'READY': + return entry + if entry['state'] not in ('LOADING', 'QUEUED'): + entry.update(state='QUEUED', error='', future=Future()) + entry['generation'] += 1 + self._put(entry) + if start: + self.start() + return entry + + def preload(self, model_ids): + # Enqueue the complete startup batch before choosing the highest-priority model. + with self.lock: + for model_id in model_ids: + self.submit(model_id, start=False) + self.start() + + def _work(self): + while True: + _, order, model_id, generation = self.queue.get() + try: + with self.lock: + entry = self.entries.get(model_id) + if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED': + continue + entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence)) + spec = entry['spec'] + future = entry['future'] + value, info = self.adapter.load(spec) + with self.lock: + entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock()) + future.set_result(True) + del value # The loader thread must not keep the last model alive after eviction. + except Exception as exc: + with self.lock: + entry.update(state='FAILED', error=self.adapter.error(exc), value=None) + if not entry['future'].done(): + entry['future'].set_exception(ValueError(entry['error'])) + finally: + self.queue.task_done() + + def acquire(self, model_id, remote=False, timeout=600): + with self.lock: + entry = self.submit(model_id) + token = uuid.uuid4().hex + entry['leases'][token] = self.clock() + 3600 if remote else float('inf') + future = entry['future'] if entry['state'] != 'READY' else None + try: + if future: + future.result(timeout=timeout) + return token + except Exception: + self.release(model_id, token) + raise + + def release(self, model_id, token): + with self.lock: + entry = self.entries.get(str(model_id)) + if entry: + entry['leases'].pop(token, None) + entry['last_used'] = self.clock() + + @contextlib.contextmanager + def inference(self, model_id): + token = self.acquire(model_id) + try: + with self.lock: + value = self.entries[str(model_id)]['value'] + yield value + finally: + self.release(model_id, token) + + def _discard(self, entry): + if entry['value'] is not None: + value = entry['value'] + self.adapter.unload(entry['spec'], value) + entry['value'] = None + del value + self.adapter.collect() + entry.update(state='UNLOADED', loaded_at=None, info={}) + + def unload(self, model_id): + with self.lock: + entry = self._entry(model_id) + self._expire_leases(entry) + if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'): + raise ValueError('模型正在使用或加载,请稍后再试') + if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']): + entry['value'] = True + self._discard(entry) + + def configure(self, model_id): + with self.lock: + entry = self._entry(model_id) + warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' + if warm_again: + entry.update(state='UNLOADED', value=None) + if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD': + self.submit(model_id) + + def _expire_leases(self, entry): + now = self.clock() + entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now} + + def sweep(self): + with self.lock: + for entry in list(self.entries.values()): + self._expire_leases(entry) + policy = entry['spec']['policy'] + # Ollama owns its idle timer; do not unload models used by other clients. + if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and + policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and + self.clock() - entry['last_used'] >= policy['idle_seconds']): + self._discard(entry) + + def status(self, model_id): + with self.lock: + entry = self._entry(model_id) + self._expire_leases(entry) + # Reconcile native residency, including expiration outside this process. + if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'): + info = self.adapter.probe(entry['spec']) + if info: + entry.update(state='READY', info=info, value=True) + elif entry['state'] == 'READY': + entry.update(state='UNLOADED', info={}, value=None, loaded_at=None) + return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'], + 'active_requests': len(entry['leases']), 'error': entry['error'], + 'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'], + 'load_order': entry['load_order'], **entry['info']} diff --git a/apps/common/model_runtime_client.py b/apps/common/model_runtime_client.py new file mode 100644 index 000000000..06b690adf --- /dev/null +++ b/apps/common/model_runtime_client.py @@ -0,0 +1,89 @@ +"""Authenticated internal model-service RPC; never forwards user-supplied endpoints.""" +import contextlib +import asyncio +import hashlib +import hmac +import json +import os +import time +import uuid +from functools import lru_cache + +import requests + + +@lru_cache(maxsize=1) +def _secret(): + if os.environ.get('SERVER_NAME') == 'local_model': + from local_model.models.system_setting import SystemSetting + else: + from system_manage.models import SystemSetting + meta = SystemSetting.objects.filter(type=1).values_list('meta', flat=True).first() + if not meta or not meta.get('value'): + raise ValueError('模型服务认证未配置') + return hashlib.sha256(('model-runtime-v1:' + meta['value']).encode()).digest() + + +def signed_request(payload): + data = {**payload, 'timestamp': time.time(), 'nonce': uuid.uuid4().hex} + body = json.dumps(data, separators=(',', ':'), sort_keys=True).encode() + signature = hmac.new(_secret(), body, hashlib.sha256).hexdigest() + return body, signature + + +def verify_request(body, signature): + expected = hmac.new(_secret(), body, hashlib.sha256).hexdigest() + if not signature or not hmac.compare_digest(expected, signature): + raise ValueError('模型服务认证失败') + data = json.loads(body) + if abs(time.time() - float(data.get('timestamp', 0))) > 60: + raise ValueError('模型服务请求已过期') + return data + + +def runtime_request(operation, model_id, **extra): + from maxkb.const import CONFIG + body, signature = signed_request({'operation': operation, 'model_id': str(model_id), **extra}) + host = CONFIG.get('LOCAL_MODEL_HOST') + url = f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{host}:{CONFIG.get("LOCAL_MODEL_PORT")}{CONFIG.get_admin_path()}/api/model_runtime' + try: + response = requests.post(url, data=body, headers={'Content-Type': 'application/json', + 'X-Model-Runtime-Signature': signature}, + timeout=(5, 620 if operation == 'acquire' else 15)) + result = response.json() + except (requests.RequestException, ValueError) as exc: + raise ValueError('本地模型服务不可用') from exc + if response.status_code != 200 or result.get('code') != 200: + raise ValueError(result.get('message') or '模型服务请求失败') + return result['data'] + + +@contextlib.contextmanager +def inference_lease(model_id): + if not model_id: + # Credential validation has no saved resource and must keep its original behavior. + yield None + return + data = runtime_request('acquire', model_id) + try: + yield data + finally: + try: + runtime_request('release', model_id, lease=data['lease']) + except ValueError: + pass # Server expires abandoned leases; never hide the inference result. + + +@contextlib.asynccontextmanager +async def async_inference_lease(model_id): + if not model_id: + yield None + return + data = await asyncio.to_thread(runtime_request, 'acquire', model_id) + try: + yield data + finally: + try: + await asyncio.to_thread(runtime_request, 'release', model_id, lease=data['lease']) + except ValueError: + pass diff --git a/apps/common/utils/table_split.py b/apps/common/utils/table_split.py new file mode 100644 index 000000000..95abdcfd3 --- /dev/null +++ b/apps/common/utils/table_split.py @@ -0,0 +1,27 @@ +"""按完整表格行分段;表头重复,重叠行不跨表/工作表。""" +def split_table_rows(header, rows, limit, overlap=0): + limit, overlap = int(limit), int(overlap) + if limit <= 0 or not 0 <= overlap < limit: + raise ValueError("Invalid table splitting limits") + chunks, current = [], [] + length = len(header) + for row in rows: + if current and length + len(row) > limit: + chunks.append({"content": header + "".join(current), "title": ""}) + carry, carried = [], 0 + if overlap: + for previous in reversed(current): + carry.insert(0, previous) + carried += len(previous) + if carried >= overlap: + break + # 必须留出至少一条新行的空间,不能只重复旧行。 + while carry and len(header) + sum(map(len, carry)) + len(row) > limit: + carry.pop(0) + current = carry + length = len(header) + sum(map(len, current)) + current.append(row) + length += len(row) + if current: + chunks.append({"content": header + "".join(current), "title": ""}) + return chunks diff --git a/apps/knowledge/embedding_params.py b/apps/knowledge/embedding_params.py new file mode 100644 index 000000000..6f1eeb329 --- /dev/null +++ b/apps/knowledge/embedding_params.py @@ -0,0 +1,65 @@ +"""Knowledge-scoped embedding options, shared by indexing and retrieval.""" +import math +import unicodedata + +from langchain_core.embeddings import Embeddings + + +def validate_params(params): + if (not isinstance(params, dict) or len(params) > 52 or + sum(key not in {'batch_size', 'normalize'} for key in params) > 50): + raise ValueError('可选参数最多50项') + result, names = {}, set() + for key, value in params.items(): + if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key): + raise ValueError('参数名称须为1~64个字符') + key = key.strip() + normalized = unicodedata.normalize('NFKC', key).casefold() + if normalized in names: + raise ValueError('参数名称不能重复') + names.add(normalized) + if value is not None and (not isinstance(value, (str, int, float, bool)) or + isinstance(value, str) and len(value) > 2000 or + isinstance(value, float) and not math.isfinite(value)): + raise ValueError('参数值须为2000字以内的文本或数值') + result[key] = value + batch = result.get('batch_size') + if batch is not None and (type(batch) is not int or not 1 <= batch <= 256): + raise ValueError('批处理数量须为1~256的整数') + if result.get('normalize') is not None and type(result['normalize']) is not bool: + raise ValueError('向量归一化参数无效') + for key in ('query_prefix', 'document_prefix'): + if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000): + raise ValueError('向量指令须为2000字以内的文本') + return {key: value for key, value in result.items() if value is not None and + (value != '' or key not in {'query_prefix', 'document_prefix'})} + + +def knowledge_params(knowledge): + params = validate_params((knowledge.meta or {}).get('embedding_params') or {}) + return {key: value for key, value in params.items() + if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}} + + +class ConfiguredEmbedding(Embeddings): + def __init__(self, model, params): + self.model = model + self.params = params + self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10 + + def _vector(self, vector): + if not self.params.get('normalize'): + return vector + norm = math.sqrt(sum(float(v) ** 2 for v in vector)) + return [float(v) / norm for v in vector] if norm else vector + + def embed_query(self, text): + return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text)) + + def embed_documents(self, texts): + prefix = self.params.get('document_prefix', '') + result = [] + for start in range(0, len(texts), self.batch_size): + result.extend(self._vector(v) for v in self.model.embed_documents( + [prefix + text for text in texts[start:start + self.batch_size]])) + return result diff --git a/apps/knowledge/migrations/0004_knowledge_rebuild_task.py b/apps/knowledge/migrations/0004_knowledge_rebuild_task.py new file mode 100644 index 000000000..edf4a19d3 --- /dev/null +++ b/apps/knowledge/migrations/0004_knowledge_rebuild_task.py @@ -0,0 +1,39 @@ +# Generated for independent knowledge index rebuild task +import uuid_utils.compat +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('knowledge', '0003_tag_documenttag'), + ('users', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='KnowledgeRebuildTask', + fields=[ + ('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')), + ('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')), + ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')), + ('status', models.CharField(choices=[('PENDING', '待执行'), ('RUNNING', '执行中'), ('SUCCESS', '成功'), ('PARTIAL', '部分成功'), ('FAILED', '失败')], db_index=True, default='PENDING', max_length=20, verbose_name='任务状态')), + ('config_snapshot', models.JSONField(default=dict, verbose_name='配置快照')), + ('document_total', models.IntegerField(default=0, verbose_name='文档总数')), + ('document_success', models.IntegerField(default=0, verbose_name='成功文档数')), + ('document_failed', models.IntegerField(default=0, verbose_name='失败文档数')), + ('error_message', models.TextField(blank=True, default='', verbose_name='失败原因')), + ('started_at', models.DateTimeField(blank=True, null=True, verbose_name='开始时间')), + ('finished_at', models.DateTimeField(blank=True, null=True, verbose_name='结束时间')), + ('celery_task_id', models.CharField(blank=True, default='', max_length=128, verbose_name='Celery任务id')), + ('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='rebuild_tasks', to='knowledge.knowledge', verbose_name='知识库')), + ('operator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='操作人')), + ], + options={ + 'db_table': 'knowledge_rebuild_task', + 'indexes': [models.Index(fields=['knowledge', 'status'], name='knowledge_r_knowled_0a6e2f_idx')], + }, + ), + ] diff --git a/apps/knowledge/migrations/0005_rebuild_task_attempt.py b/apps/knowledge/migrations/0005_rebuild_task_attempt.py new file mode 100644 index 000000000..837def6ed --- /dev/null +++ b/apps/knowledge/migrations/0005_rebuild_task_attempt.py @@ -0,0 +1,17 @@ +# Generated for rebuild task attempt polling +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('knowledge', '0004_knowledge_rebuild_task'), + ] + + operations = [ + migrations.AddField( + model_name='knowledgerebuildtask', + name='attempt', + field=models.IntegerField(default=0, verbose_name='汇总轮询次数'), + ), + ] diff --git a/apps/knowledge/serializers/knowledge_ext.py b/apps/knowledge/serializers/knowledge_ext.py new file mode 100644 index 000000000..495dd73eb --- /dev/null +++ b/apps/knowledge/serializers/knowledge_ext.py @@ -0,0 +1,215 @@ +# coding=utf-8 +"""知识库扩展配置序列化器:业务元数据、分段规则。""" +from collections.abc import Mapping +import re + +from django.db import transaction +from django.db.models import QuerySet +from django.utils import timezone +from rest_framework import serializers + +from common.exception.app_exception import AppApiException +from knowledge.models import Knowledge + +METADATA_FIELDS = [ + {"field": "business_domain", "label": "业务领域"}, + {"field": "data_source", "label": "数据来源"}, + {"field": "owner_dept", "label": "责任部门"}, + {"field": "business_version", "label": "业务版本"}, +] +METADATA_MAX_LEN = 64 + + +def _fmt(value): + if not value: + return "" + try: + value = timezone.localtime(value) + except Exception: + pass + return value.strftime("%Y-%m-%d %H:%M:%S") + + +def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge: + query = QuerySet(Knowledge) + if lock: + query = query.select_for_update() + knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first() + if knowledge is None: + raise AppApiException(500, "知识库不存在") + return knowledge + + +def _creator_candidates(workspace_id): + from users.models import User, UserActivationApplication + from common.database_model_manage.database_model_manage import DatabaseModelManage + from django.db.models import Q + pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True) + query = User.objects.filter(is_active=True).exclude(id__in=pending) + membership = DatabaseModelManage.get_model('workspace_user_role_mapping') + if membership is not None: + ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True) + query = query.filter(Q(id__in=ids) | Q(role='ADMIN')) + return query + + +def _metadata_fields(business): + # An explicit empty list means all custom fields were removed; do not revive old fields. + if isinstance(business.get('custom_fields'), list): + return business['custom_fields'] + return [{'name': item['label'], 'value': business[item['field']]} + for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')] + + +def _validate_custom_fields(items): + import unicodedata + if not isinstance(items, list) or len(items) > 50: + raise AppApiException(500, '元数据最多50项') + reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'} + result, names = [], set() + for item in items: + if not isinstance(item, Mapping) or set(item) != {'name', 'value'}: + raise AppApiException(500, '元数据需包含名称和值') + name, value = item['name'], item['value'] + if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name): + raise AppApiException(500, '元数据名称须为1~64个字符') + name = name.strip() + normalized = unicodedata.normalize('NFKC', name).casefold() + if normalized in reserved: + raise AppApiException(500, '不能使用固定属性名称') + if normalized in names: + raise AppApiException(500, '元数据名称不能重复') + if not isinstance(value, str) or len(value) > 2000: + raise AppApiException(500, '元数据值须为2000字以内的文本') + names.add(normalized) + result.append({'name': name, 'value': value.strip()}) + return result + + +class KnowledgeMetadataSerializer(serializers.Serializer): + """Fixed system attributes plus freely named business metadata; save atomically.""" + + @staticmethod + def one(workspace_id: str, knowledge_id: str): + from users.models import User + knowledge = _get_knowledge(workspace_id, knowledge_id) + data = (knowledge.meta or {}).get('business_metadata') or {} + creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None + candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False} + for u in _creator_candidates(workspace_id).order_by('nick_name')] + if creator and not any(u['id'] == str(creator.id) for u in candidates): + candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True}) + return { + 'knowledge_id': str(knowledge.id), + 'user_id': str(knowledge.user_id) if knowledge.user_id else '', + 'creator_name': (creator.nick_name or creator.username) if creator else '', + 'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time), + 'creator_options': candidates, + 'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)], + 'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '', + } + + @staticmethod + @transaction.atomic + def update(workspace_id: str, knowledge_id: str, user, data: dict): + if not isinstance(data, Mapping): + raise AppApiException(500, '元数据必须为属性对象') + legacy = {item['field']: item['label'] for item in METADATA_FIELDS} + if set(data) - (set(legacy) | {'fields', 'user_id'}): + raise AppApiException(500, '包含不支持的元数据属性') + if 'fields' in data and set(data) & set(legacy): + raise AppApiException(500, '不能混用新旧元数据格式') + fields = _validate_custom_fields(data['fields']) if 'fields' in data else None + values = {} + for key, label in legacy.items(): + if key not in data: + continue + value = '' if data[key] is None else data[key] + if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN: + raise AppApiException(500, f'{label}须为64字以内的文本') + values[key] = value.strip() + knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True) + if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id): + try: + creator_id = serializers.UUIDField().run_validation(data['user_id']) + except serializers.ValidationError: + raise AppApiException(500, '请选择有效创建者') + if not _creator_candidates(workspace_id).filter(id=creator_id).exists(): + raise AppApiException(500, '创建者须为已激活且启用的成员') + knowledge.user_id = creator_id + meta = dict(knowledge.meta or {}) + business = dict(meta.get('business_metadata') or {}) + if fields is not None: + business['custom_fields'] = fields + elif 'custom_fields' in business: + current = list(_metadata_fields(business)) + for key, value in values.items(): + existing = next((item for item in current if item['name'] == legacy[key]), None) + if existing is not None: + existing['value'] = value + elif value: + current.append({'name': legacy[key], 'value': value}) + business['custom_fields'] = _validate_custom_fields(current) + business.update(values) + business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username, + updated_at=_fmt(timezone.now())) + meta['business_metadata'] = business + knowledge.meta = meta + knowledge.save(update_fields=['meta', 'user', 'update_time']) + return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id) + + +class KnowledgeSegmentationSerializer(serializers.Serializer): + """知识库分段规则:查看 / 修改。 + + chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度; + 保存后新建文档上传时默认使用该规则,上传时可覆盖。 + """ + + DEFAULTS = {"chunk_size": 4096, "chunk_overlap": 0} + + @staticmethod + def one(workspace_id: str, knowledge_id: str): + knowledge = _get_knowledge(workspace_id, knowledge_id) + data = dict((knowledge.meta or {}).get('segmentation') or {}) + result = { + "chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"], + "chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None + else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"], + } + result.update({ + "updated_by": data.get("updated_by_name") or "", + "updated_at": data.get("updated_at") or "", + "knowledge_id": str(knowledge.id), + }) + return result + + @staticmethod + @transaction.atomic + def update(workspace_id: str, knowledge_id: str, user, data: dict): + if not isinstance(data, Mapping): + raise AppApiException(500, "分段规则必须为属性对象") + def integer(value): + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise AppApiException(500, "分段长度/重叠长度必须为整数") + if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value): + raise AppApiException(500, "分段长度/重叠长度必须为整数") + return int(value) + chunk_size = integer(data.get("chunk_size")) + chunk_overlap = integer(data.get("chunk_overlap", 0)) + if chunk_size < 50 or chunk_size > 100000: + raise AppApiException(500, "分段长度取值范围为 50~100000") + if chunk_overlap < 0 or chunk_overlap >= chunk_size: + raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度") + knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True) + meta = dict(knowledge.meta or {}) + meta['segmentation'] = { + "chunk_size": chunk_size, + "chunk_overlap": chunk_overlap, + "updated_by": str(user.id), + "updated_by_name": getattr(user, 'nick_name', '') or user.username, + "updated_at": _fmt(timezone.now()), + } + knowledge.meta = meta + knowledge.save(update_fields=["meta", "update_time"]) + return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id) diff --git a/apps/knowledge/serializers/rebuild.py b/apps/knowledge/serializers/rebuild.py new file mode 100644 index 000000000..f7e470a82 --- /dev/null +++ b/apps/knowledge/serializers/rebuild.py @@ -0,0 +1,129 @@ +# coding=utf-8 +"""知识库独立索引重建任务序列化器。""" +from django.db import transaction +from django.utils import timezone +from rest_framework import serializers + +from common.exception.app_exception import AppApiException +from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document) +from knowledge.task.rebuild import rebuild_knowledge_index +from models_provider.models import Model + + +def _fmt(value): + if not value: + return "" + try: + value = timezone.localtime(value) + except Exception: + pass + return value.strftime("%Y-%m-%d %H:%M:%S") + + +def serialize_task(task: KnowledgeRebuildTask): + total = task.document_total or 0 + success = task.document_success or 0 + failed = task.document_failed or 0 + processed = success + failed + progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0) + status_label = dict(RebuildStatus.choices).get(task.status, task.status) + return { + "id": str(task.id), + "knowledge_id": str(task.knowledge_id), + "workspace_id": task.workspace_id, + "status": task.status, + "status_label": status_label, + "config_snapshot": task.config_snapshot or {}, + "document_total": total, + "document_success": success, + "document_failed": failed, + "progress": progress, + "error_message": task.error_message or "", + "created_at": _fmt(task.create_time), + "started_at": _fmt(task.started_at), + "finished_at": _fmt(task.finished_at), + } + + +class KnowledgeRebuildSerializer(serializers.Serializer): + @staticmethod + def create(workspace_id: str, knowledge_id: str, user_id: str): + # 锁住知识库行,串行化同一知识库的检查和任务创建。 + with transaction.atomic(): + knowledge = Knowledge.objects.select_for_update().filter( + id=knowledge_id, workspace_id=workspace_id + ).first() + if knowledge is None: + raise AppApiException(500, "知识库不存在") + if KnowledgeRebuildTask.objects.filter( + knowledge_id=knowledge_id, + status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING] + ).exists(): + raise AppApiException(500, "该知识库已有重建任务在执行") + model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None + if model is None: + raise AppApiException(500, "知识库未配置可用的向量模型") + document_ids = [str(pk) for pk in Document.objects.filter( + knowledge_id=knowledge_id).values_list("id", flat=True)] + if not document_ids: + raise AppApiException(500, "知识库没有可重建的文档") + snapshot = { + "embedding_model_id": str(model.id), + "embedding_params": (knowledge.meta or {}).get("embedding_params", {}), + "embedding_model_name": model.name, + "embedding_model": model.model_name, + "knowledge_name": knowledge.name, + "document_ids": document_ids, + "document_total": len(document_ids), + "snapshot_at": _fmt(timezone.now()), + } + task = KnowledgeRebuildTask.objects.create( + knowledge_id=knowledge_id, workspace_id=workspace_id, + status=RebuildStatus.PENDING, config_snapshot=snapshot, + document_total=len(document_ids), operator_id=user_id, + ) + + def dispatch(): + try: + result = rebuild_knowledge_index.delay(str(task.id)) + # 不覆盖 worker 已经更新的状态。 + KnowledgeRebuildTask.objects.filter(id=task.id).update( + celery_task_id=getattr(result, "id", "") or "") + except Exception as exc: + KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update( + status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}", + finished_at=timezone.now(), update_time=timezone.now()) + + transaction.on_commit(dispatch) + task.refresh_from_db() + return serialize_task(task) + + @staticmethod + def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int): + query_set = KnowledgeRebuildTask.objects.filter( + knowledge_id=knowledge_id, workspace_id=workspace_id + ).order_by("-create_time") + total = query_set.count() + start = max(0, (current_page - 1) * page_size) + records = [serialize_task(item) for item in query_set[start:start + page_size]] + return {"records": records, "total": total, "current_page": current_page, "page_size": page_size} + + @staticmethod + def one(workspace_id: str, knowledge_id: str, task_id: str): + task = KnowledgeRebuildTask.objects.filter( + id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id + ).first() + if task is None: + raise AppApiException(500, "重建任务不存在") + return serialize_task(task) + + @staticmethod + def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str): + old_task = KnowledgeRebuildTask.objects.filter( + id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id + ).first() + if old_task is None: + raise AppApiException(500, "重建任务不存在") + if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL): + raise AppApiException(500, "仅失败或部分成功的任务可以重新发起") + return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id) diff --git a/apps/knowledge/task/rebuild.py b/apps/knowledge/task/rebuild.py new file mode 100644 index 000000000..faea081b7 --- /dev/null +++ b/apps/knowledge/task/rebuild.py @@ -0,0 +1,146 @@ +# coding=utf-8 +"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。""" +from django.db import transaction +from django.utils import timezone + +from common.db.sql_execute import sql_execute +from common.event.listener_manage import ListenerManagement +from common.utils.logger import maxkb_logger +from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph, + TaskType, State, Status) +from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index +from ops import celery_app + +POLL_SECONDS = 5 +MAX_POLLS = 120 + + +def _index_exists(knowledge_id: str) -> bool: + rows = sql_execute( + "SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s", + [f"embedding_hnsw_idx_{knowledge_id}"] + ) + return bool(rows) + + +def _fail(task_id, message): + KnowledgeRebuildTask.objects.filter( + id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING] + ).update(status=RebuildStatus.FAILED, error_message=message, + finished_at=timezone.now(), update_time=timezone.now()) + + +@celery_app.task(name="celery:rebuild_knowledge_index") +def rebuild_knowledge_index(task_id: str): + # 重复投递不能重复清空向量或重复派发。 + claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update( + status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="", + attempt=0, update_time=timezone.now()) + if not claimed: + return + try: + task = KnowledgeRebuildTask.objects.get(id=task_id) + knowledge = Knowledge.objects.filter(id=task.knowledge_id).first() + if knowledge is None: + raise ValueError("知识库不存在") + snapshot = task.config_snapshot or {} + model_id = snapshot.get("embedding_model_id") + if not model_id or str(knowledge.embedding_model_id) != model_id: + raise ValueError("向量模型已变更,请重新发起任务") + if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}): + raise ValueError('嵌入参数已变更,请重新发起任务') + # 兼容历史任务;新任务在受理时即固定文档范围。 + document_ids = snapshot.get("document_ids") + if document_ids is None: + document_ids = [str(pk) for pk in Document.objects.filter( + knowledge_id=knowledge.id).values_list("id", flat=True)] + snapshot["document_ids"] = document_ids + task.config_snapshot = snapshot + task.save(update_fields=["config_snapshot", "update_time"]) + documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)) + if not documents or len(documents) != len(document_ids): + raise ValueError("重建范围为空或文档已删除,请重新发起任务") + if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents): + raise ValueError("文档正在向量化,请完成后重新发起") + + ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id)) + drop_knowledge_index(knowledge_id=str(knowledge.id)) + task.document_total = len(document_ids) + task.save(update_fields=["document_total", "update_time"]) + # 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。 + target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids) + ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING) + ListenerManagement.update_status( + Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids), + TaskType.EMBEDDING, State.PENDING) + from knowledge.task.embedding import embedding_by_document + dispatch_errors = [] + for document in documents: + try: + embedding_by_document.delay(str(document.id), model_id) + except Exception as exc: + ListenerManagement.update_status( + Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE) + dispatch_errors.append(f"{document.name}:{exc}") + if dispatch_errors: + task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000] + task.save(update_fields=["error_message", "update_time"]) + # 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。 + finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS) + except Exception as exc: + _fail(task_id, f"重建任务启动失败:{exc}") + maxkb_logger.exception("rebuild knowledge index failed: %s", task_id) + + +@celery_app.task(name="celery:finalize_rebuild_task") +def finalize_rebuild_task(task_id: str): + try: + with transaction.atomic(): + task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first() + if task is None or task.status != RebuildStatus.RUNNING: + return + snapshot = task.config_snapshot or {} + document_ids = snapshot.get("document_ids", []) + documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids)) + states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents] + success = states.count(State.SUCCESS) + failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED) + for state in states) + len(document_ids) - len(documents) + pending = len(document_ids) - success - failed + task.document_total = len(document_ids) + task.document_success = success + task.document_failed = failed + if pending: + task.attempt += 1 + if task.attempt < MAX_POLLS: + task.save(update_fields=["document_total", "document_success", "document_failed", + "attempt", "update_time"]) + transaction.on_commit(lambda: finalize_rebuild_task.apply_async( + (task_id,), countdown=POLL_SECONDS)) + return + task.status = RebuildStatus.FAILED + task.error_message = f"重建超时:仍有 {pending} 个文档未完成" + else: + knowledge_id = str(task.knowledge_id) + knowledge = Knowledge.objects.filter(id=task.knowledge_id).first() + if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or + snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})): + raise ValueError("向量模型已变更,请重新发起任务") + ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)() + create_knowledge_index(knowledge_id=knowledge_id) + index_ready = _index_exists(knowledge_id) + if success and failed == 0 and index_ready: + task.status = RebuildStatus.SUCCESS + task.error_message = "" + elif success and failed and index_ready: + task.status = RebuildStatus.PARTIAL + task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message + else: + task.status = RebuildStatus.FAILED + task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message + task.finished_at = timezone.now() + task.save(update_fields=["document_total", "document_success", "document_failed", "status", + "error_message", "finished_at", "attempt", "update_time"]) + except Exception as exc: + _fail(task_id, f"重建结果汇总失败:{exc}") + maxkb_logger.exception("finalize rebuild task failed: %s", task_id) diff --git a/apps/knowledge/tasks.py b/apps/knowledge/tasks.py new file mode 100644 index 000000000..8b61fd48a --- /dev/null +++ b/apps/knowledge/tasks.py @@ -0,0 +1,3 @@ +"""Celery 任务发现入口,确保独立重建和向量化任务被 Worker 注册。""" +from knowledge.task import embedding # noqa: F401 +from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task # noqa: F401 diff --git a/apps/knowledge/test_embedding_params.py b/apps/knowledge/test_embedding_params.py new file mode 100644 index 000000000..b2af2d073 --- /dev/null +++ b/apps/knowledge/test_embedding_params.py @@ -0,0 +1,73 @@ +import unittest +from types import SimpleNamespace + +from knowledge.embedding_params import ConfiguredEmbedding, validate_params, knowledge_params + + +class RecordingModel: + batch_size = 10 + + def __init__(self): + self.calls = [] + + def embed_query(self, text): + self.calls.append(('query', text)) + return [3., 4.] + + def embed_documents(self, texts): + self.calls.append(('documents', texts)) + return [[3., 4.] for _ in texts] + + +class EmbeddingParamsTests(unittest.TestCase): + def test_inference_uses_separate_instructions_and_same_normalization(self): + base = RecordingModel() + model = ConfiguredEmbedding(base, {'normalize': True, 'query_prefix': '检索:', 'document_prefix': '正文:'}) + self.assertEqual(model.embed_query('内容'), [.6, .8]) + self.assertEqual(model.embed_documents(['内容']), [[.6, .8]]) + self.assertEqual(base.calls, [('query', '检索:内容'), ('documents', ['正文:内容'])]) + + def test_batch_size_is_applied_to_real_calls(self): + base = RecordingModel() + model = ConfiguredEmbedding(base, {'batch_size': 2}) + self.assertEqual(len(model.embed_documents(['a', 'b', 'c', 'd', 'e'])), 5) + self.assertEqual([len(call[1]) for call in base.calls], [2, 2, 1]) + + def test_knowledge_overrides_do_not_leak_between_shared_models(self): + base = RecordingModel() + first = ConfiguredEmbedding(base, {'query_prefix': 'A'}) + second = ConfiguredEmbedding(base, {'query_prefix': 'B'}) + first.embed_query('x'); second.embed_query('x'); first.embed_query('x') + self.assertEqual(base.calls, [('query', 'Ax'), ('query', 'Bx'), ('query', 'Ax')]) + self.assertEqual(base.batch_size, 10) + + def test_invalid_inputs_are_rejected(self): + for params in ([], {'batch_size': True}, {'batch_size': 0}, {'batch_size': 257}, + {'batch_size': 2.5}, {'normalize': 'true'}, {'query_prefix': 1}, + {'document_prefix': 'x' * 2001}, {'': 'value'}, {'custom': {}}, + {'custom': ['value']}, {'x': 1, 'X': 2}, {'custom': float('nan')}, + {'a' * 65: 'value'}, {str(i): 'value' for i in range(51)}): + with self.subTest(params=params), self.assertRaises(ValueError): + validate_params(params) + + def test_custom_names_are_saved_but_unsupported_options_do_not_reach_inference(self): + params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'} + self.assertEqual(validate_params(params), params) + active = knowledge_params(SimpleNamespace(meta={'embedding_params': params})) + self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:'}) + base = RecordingModel() + model = ConfiguredEmbedding(base, active) + model.embed_query('问题') + self.assertEqual(base.calls, [('query', '检索:问题')]) + + def test_legacy_and_empty_params_keep_original_behavior(self): + self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {}) + self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {}) + base = RecordingModel() + self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [3., 4.]) + self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), []) + + def test_zero_vector_is_safe(self): + base = RecordingModel() + base.embed_query = lambda _: [0., 0.] + self.assertEqual(ConfiguredEmbedding(base, {'normalize': True}).embed_query('x'), [0., 0.]) diff --git a/apps/knowledge/test_knowledge_ext.py b/apps/knowledge/test_knowledge_ext.py new file mode 100644 index 000000000..99e8a504d --- /dev/null +++ b/apps/knowledge/test_knowledge_ext.py @@ -0,0 +1,288 @@ +"""Real isolated persistence and document parsing regression checks.""" +import io +import importlib +import unittest +from types import SimpleNamespace +from unittest.mock import patch, Mock +from django.test import TestCase +from django.core.files.uploadedfile import SimpleUploadedFile +from common.exception.app_exception import AppApiException +from common.utils.split_model import smart_split_paragraph, SplitModel +from common.utils.table_split import split_table_rows +from knowledge.models import KnowledgeFolder, Knowledge +from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules + + +class MetadataTests(TestCase): + def setUp(self): + KnowledgeFolder.objects.create(id='default', name='root') + self.k = Knowledge.objects.create(name='test', desc='', folder_id='default', + meta={'internal': {'preserve': True}}) + self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester') + + def test_metadata_persists_and_preserves_other_settings(self): + Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '}) + self.k.refresh_from_db() + self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维') + self.assertEqual(self.k.meta['internal'], {'preserve': True}) + result = Metadata.one('default', str(self.k.id)) + self.assertEqual(len(result['fields']), 1) + self.assertEqual(result['updated_by'], '测试员') + self.assertTrue(result['updated_at']) + + def test_partial_metadata_update_does_not_clear_other_fields(self): + Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'}) + Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'}) + self.k.refresh_from_db() + self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维') + + def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self): + Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'}) + old = Metadata.one('default', str(self.k.id)) + self.assertEqual(old['fields'][0]['name'], '业务领域') + Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]}) + self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')]) + Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]}) + self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码') + Metadata.update('default', str(self.k.id), self.user, {'fields': []}) + self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], []) + self.k.refresh_from_db() + self.assertEqual(self.k.meta['internal'], {'preserve': True}) + + def test_custom_field_validation_does_not_write(self): + cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}], + [{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}], + [{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}], + [{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}], + [{'name': str(i), 'value': ''} for i in range(51)]] + for fields in cases: + with self.subTest(fields=fields), self.assertRaises(AppApiException): + Metadata.update('default', str(self.k.id), self.user, {'fields': fields}) + with self.assertRaises(AppApiException): + Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'}) + self.k.refresh_from_db() + self.assertNotIn('business_metadata', self.k.meta) + + def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self): + from users.models import User + creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True) + original_time = self.k.create_time + result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]}) + self.k.refresh_from_db() + self.assertEqual(self.k.user_id, creator.id) + self.assertEqual(self.k.create_time, original_time) + self.assertEqual(result['creator_name'], '新创建者') + self.assertEqual(result['user_id'], str(creator.id)) + self.assertTrue(result['update_time']) + self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']]) + + def test_rejected_pending_disabled_creators_cannot_be_selected(self): + from users.models import User, UserActivationApplication + creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True) + application = UserActivationApplication.objects.create(user=creator, status='PENDING') + for status in ('PENDING', 'REJECTED'): + application.status = status; application.save() + with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]}) + self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']]) + application.status = 'APPROVED'; application.save() + creator.is_active = False; creator.save() + with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)}) + self.k.refresh_from_db() + self.assertIsNone(self.k.user_id) + self.assertNotIn('business_metadata', self.k.meta) + + def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self): + from knowledge.serializers.knowledge import KnowledgeSerializer + other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={}) + operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'}) + operation._validated_data = operation.initial_data + operation._errors = {} + params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'} + with patch.object(operation, 'is_valid'): + operation.edit({'embedding_params': params}, select_one=False) + operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False) + self.k.refresh_from_db(); other.refresh_from_db() + self.assertEqual(self.k.meta['embedding_params'], params) + self.assertNotIn('embedding_params', other.meta) + with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException): + operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False) + self.k.refresh_from_db() + self.assertEqual(self.k.meta['embedding_params'], params) + + def test_general_settings_cannot_restore_stale_metadata(self): + from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest + Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]}) + self.k.refresh_from_db() + protected = self.k.meta['business_metadata'] + operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'}) + operation._validated_data = operation.initial_data + operation._errors = {} + with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'): + operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False) + self.k.refresh_from_db() + self.assertEqual(self.k.meta['business_metadata'], protected) + self.assertEqual(self.k.meta['selector'], 'body') + self.assertNotIn('parser_id', self.k.meta) + + def test_invalid_types_and_unknown_fields_do_not_write(self): + for data in ([1], {'business_domain': 123}, {'business_domain': False}, + {'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}): + with self.subTest(data=data), self.assertRaises(AppApiException): + Metadata.update('default', str(self.k.id), self.user, data) + self.k.refresh_from_db() + self.assertNotIn('business_metadata', self.k.meta) + + def test_workspace_boundary(self): + for fn in (Metadata.one, Rules.one): + with self.assertRaises(AppApiException): + fn('other', str(self.k.id)) + + def test_segmentation_and_metadata_updates_preserve_each_other(self): + Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20}) + Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'}) + self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20) + Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0}) + self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维') + + def test_segmentation_rejects_invalid_and_fractional_values(self): + for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10), + (100,1.2), (True,0), (100,False), ('100.5',0)): + with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException): + Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap}) + + def test_upload_inherits_defaults_and_accepts_explicit_override(self): + from knowledge.serializers.document import DocumentSerializers + Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20}) + serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'}) + with patch.object(serializer,'file_to_paragraph',return_value=[]) as split: + f=SimpleUploadedFile('test.txt',b'test') + serializer.parse({'file':[f]}) + self.assertEqual(split.call_args.args[-2:], (100,20)) + serializer.parse({'file':[f],'limit':'200','overlap':'0'}) + self.assertEqual(split.call_args.args[-2:], (200,0)) + with self.assertRaises(AppApiException): + serializer.parse({'file':[f],'limit':'100','overlap':'100'}) + + +class ParserTests(unittest.TestCase): + text=''.join(chr(0x4e00+i) for i in range(500)) + + def check_overlap(self, parts): + self.assertGreater(len(parts),1) + self.assertTrue(all(len(p)<=100 for p in parts)) + self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1))) + + def test_plain_text_overlap(self): + self.check_overlap(smart_split_paragraph(self.text,100,20)) + + def test_pdf_toc_and_internal_link_calls_pass_overlap(self): + from common.handle.impl.text.pdf_split_handle import PdfSplitHandle + doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1, + load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text)) + result=PdfSplitHandle.handle_toc(doc,100,20) + self.check_overlap([p['content'] for p in result]) + # 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。 + import ast,inspect,textwrap + for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links): + calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method)))) + if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph'] + self.assertTrue(calls) + self.assertTrue(all(len(n.args)==3 for n in calls)) + + def test_table_overlap_preserves_rows_headers_and_termination(self): + header='| id | value |\n| --- | --- |\n' + rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)] + chunks=split_table_rows(header,iter(rows),120,20) + self.assertGreater(len(chunks),1) + self.assertLess(len(chunks),len(rows)) + self.assertTrue(all(c['content'].startswith(header) for c in chunks)) + self.assertTrue(all(len(c['content'])<=120 for c in chunks)) + for a,b in zip(chunks,chunks[1:]): + self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) + for row in rows:self.assertTrue(any(row in c['content'] for c in chunks)) + + def test_oversized_table_row_is_not_truncated(self): + row='| '+'x'*500+' |\n' + result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20) + self.assertTrue(any(row in p['content'] for p in result)) + self.assertIn('last',result[-1]['content']) + + def test_csv_and_xlsx_use_row_overlap(self): + import openpyxl + from common.handle.impl.text.csv_split_handle import CsvSplitHandle + from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle + csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode() + book=openpyxl.Workbook();sheet=book.active + for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row) + out=io.BytesIO();book.save(out) + for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]: + f=SimpleUploadedFile(name,data) + result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20) + chunks=result[0]['content'] if isinstance(result,list) else result['content'] + self.assertGreater(len(chunks),1) + for a,b in zip(chunks,chunks[1:]): + self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) + + def test_xls_uses_row_overlap(self): + from common.handle.impl.text.xls_split_handle import handle_sheet + data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)] + sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i]) + chunks=handle_sheet('test',sheet,120,20)['content'] + for a,b in zip(chunks,chunks[1:]): + self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:]) + + def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self): + migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form') + model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock()) + manager=Mock();manager.using.return_value.filter.return_value=[model] + apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager)) + editor=SimpleNamespace(connection=SimpleNamespace(alias='default')) + migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor) + self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p']) + model.save.assert_called_once() + + def test_generation_parameters_reach_stream_and_nonstream_http_body(self): + import json + import httpx + from openai import OpenAI + from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel + from langchain_core.messages import HumanMessage + requests=[] + def transport(request): + body=json.loads(request.content);requests.append(body) + if body.get('stream'): + events=[] + for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]: + item={'id':'test','object':'chat.completion.chunk','model':'test','created':1, + 'choices':[{'index':0,'delta':delta,'finish_reason':finish}]} + events.append('data: '+json.dumps(item)+'\n\n') + return httpx.Response(200,headers={'Content-Type':'text/event-stream'}, + text=''.join(events)+'data: [DONE]\n\n') + return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1, + 'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}], + 'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}}) + with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http: + client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http) + model=BaiLianChatModel.new_instance('LLM','test',{ + 'api_base':'http://example.invalid/v1','api_key':'test-only'}, + temperature=0.23,top_p=0.42,max_tokens=10) + model.client=client.chat.completions + model.root_client=client + model.streaming=False + self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok') + model.streaming=True + self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok') + self.assertEqual(len(requests),2) + self.assertEqual([bool(r.get('stream')) for r in requests],[False,True]) + for req in requests: + self.assertEqual(req['temperature'],0.23) + self.assertEqual(req['top_p'],0.42) + + def test_new_knowledge_tasks_use_isolated_queue(self): + from ops import celery_app + import knowledge.tasks + for name in ('celery:embedding_by_document','celery:embedding_by_knowledge', + 'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'): + self.assertIn(name,celery_app.tasks) + route=celery_app.amqp.router.route({},name) + self.assertEqual(route['queue'].name,'uniskb_knowledge') diff --git a/apps/knowledge/test_rebuild.py b/apps/knowledge/test_rebuild.py new file mode 100644 index 000000000..dd598d4ec --- /dev/null +++ b/apps/knowledge/test_rebuild.py @@ -0,0 +1,187 @@ +"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls.""" +import unittest +import uuid +from types import SimpleNamespace +from unittest.mock import patch, Mock +from contextlib import ExitStack +from django.db import transaction, connection +from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State +from knowledge.serializers.rebuild import KnowledgeRebuildSerializer +from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS +from common.exception.app_exception import AppApiException + + +class RebuildTests(unittest.TestCase): + def setUp(self): + with connection.cursor() as cursor: + cursor.execute('DELETE FROM knowledge_rebuild_task') + self.kid = str(uuid.uuid4()) + self.docid = str(uuid.uuid4()) + self.modelid = str(uuid.uuid4()) + self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test') + self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2') + self.stack = ExitStack() + self.addCleanup(self.stack.close) + def mock(name): + return self.stack.enter_context(patch(name)) + self.k = mock('knowledge.serializers.rebuild.Knowledge.objects') + self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge + self.k.filter.return_value.first.return_value = self.knowledge + self.docs = mock('knowledge.serializers.rebuild.Document.objects') + self.docs.filter.return_value.values_list.return_value = [self.docid] + self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document]) + self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects') + self.models = mock('knowledge.serializers.rebuild.Model.objects') + self.models.filter.return_value.first.return_value = SimpleNamespace( + id=self.modelid, name='embedding', model_name='test-model') + self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay') + self.delay.return_value.id = 'queued-test' + self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async') + self.listener = mock('knowledge.task.rebuild.ListenerManagement') + self.drop = mock('knowledge.task.rebuild.drop_knowledge_index') + self.create_index = mock('knowledge.task.rebuild.create_knowledge_index') + self.index_exists = mock('knowledge.task.rebuild._index_exists') + self.index_exists.return_value = True + self.child = mock('knowledge.task.embedding.embedding_by_document.delay') + + def create(self): + return KnowledgeRebuildSerializer.create('default', self.kid, None) + + def task(self, status=RebuildStatus.RUNNING): + return KnowledgeRebuildTask.objects.create( + knowledge_id=self.kid, workspace_id='default', status=status, document_total=1, + config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]}) + + def test_dispatch_only_after_commit(self): + with transaction.atomic(): + result = self.create() + self.delay.assert_not_called() + self.assertEqual(result['status'], RebuildStatus.PENDING) + self.delay.assert_called_once() + self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test') + + def test_rollback_does_not_dispatch(self): + with self.assertRaises(ValueError): + with transaction.atomic(): + self.create() + raise ValueError('rollback') + self.delay.assert_not_called() + self.assertFalse(KnowledgeRebuildTask.objects.exists()) + + def test_broker_failure_persists_failed_record(self): + self.delay.side_effect = RuntimeError('broker offline') + result = self.create() + self.assertEqual(result['status'], RebuildStatus.FAILED) + self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message) + + def test_active_task_rejected_under_knowledge_lock(self): + self.task() + with self.assertRaises(AppApiException): + self.create() + self.k.select_for_update.assert_called_once() + self.assertEqual(KnowledgeRebuildTask.objects.count(), 1) + + def test_empty_knowledge_rejected(self): + self.docs.filter.return_value.values_list.return_value = [] + with self.assertRaises(AppApiException): + self.create() + self.assertFalse(KnowledgeRebuildTask.objects.exists()) + + def test_success_cannot_be_retried(self): + task = self.task(RebuildStatus.SUCCESS) + with self.assertRaises(AppApiException): + KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None) + + def test_failed_retry_creates_new_record(self): + old = self.task(RebuildStatus.FAILED) + result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None) + self.assertNotEqual(result['id'], str(old.id)) + self.assertEqual(KnowledgeRebuildTask.objects.count(), 2) + + def test_duplicate_worker_delivery_does_not_delete_vectors(self): + task = self.task() + rebuild_knowledge_index.run(str(task.id)) + self.listener.delete_embedding_by_knowledge.assert_not_called() + + def test_worker_dispatches_independent_monitor(self): + task = self.task(RebuildStatus.PENDING) + rebuild_knowledge_index.run(str(task.id)) + self.child.assert_called_once_with(self.docid, self.modelid) + self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid]) + self.poll.assert_called_once() + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.RUNNING) + + def test_document_dispatch_failure_is_counted(self): + task = self.task(RebuildStatus.PENDING) + self.child.side_effect = RuntimeError('cannot queue') + rebuild_knowledge_index.run(str(task.id)) + self.listener.update_status.assert_any_call( + self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE) + self.document.status = 'nn3' + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.FAILED) + self.assertEqual(task.document_failed, 1) + self.assertIn('cannot queue', task.error_message) + + def test_success_requires_index(self): + task = self.task() + self.index_exists.return_value = False + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.FAILED) + + def test_successful_finalization(self): + task = self.task() + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.SUCCESS) + self.assertEqual(task.document_success, 1) + self.assertIsNotNone(task.finished_at) + + def test_pending_persists_progress_and_schedules_next_poll(self): + task = self.task() + other = SimpleNamespace(id=uuid.uuid4(), status='nn0') + task.config_snapshot['document_ids'].append(str(other.id)) + task.save() + self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other]) + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.RUNNING) + self.assertEqual(task.document_success, 1) + self.poll.assert_called_once() + + def test_timeout_records_failure(self): + task = self.task() + task.attempt = MAX_POLLS - 1 + task.save() + self.document.status = 'nn0' + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.FAILED) + self.assertIn('超时', task.error_message) + + def test_ignored_document_is_failure(self): + task = self.task() + self.document.status = 'nnn' + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.document_failed, 1) + self.assertEqual(task.status, RebuildStatus.FAILED) + + def test_deleted_document_is_not_silent_success(self): + task = self.task() + self.docs.filter.return_value.__iter__.side_effect = lambda: iter([]) + finalize_rebuild_task.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.document_failed, 1) + self.assertEqual(task.status, RebuildStatus.FAILED) + + def test_model_change_fails_before_destructive_work(self): + task = self.task(RebuildStatus.PENDING) + self.knowledge.embedding_model_id = str(uuid.uuid4()) + rebuild_knowledge_index.run(str(task.id)) + task.refresh_from_db() + self.assertEqual(task.status, RebuildStatus.FAILED) + self.listener.delete_embedding_by_knowledge.assert_not_called() diff --git a/apps/local_model/runtime.py b/apps/local_model/runtime.py new file mode 100644 index 000000000..622b67afc --- /dev/null +++ b/apps/local_model/runtime.py @@ -0,0 +1,172 @@ +"""Weight owner for local resources; shared by both assistants through model IDs.""" +import gc +import hashlib +import json +import logging +import threading +import time +from urllib.parse import urlsplit, urlunsplit + +import requests +from django.db import close_old_connections + +from common.model_runtime import ModelRuntime, loading_policy + +logger = logging.getLogger('maxkb') +SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}} + + +def driver_for(provider, model_type): + driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider) + return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None + + +def model_spec(model_id): + from local_model.models import Model + from local_model.serializers.rsa_util import rsa_long_decrypt + try: + model = Model.objects.get(id=model_id, status='SUCCESS') + except Model.DoesNotExist: + raise ValueError('模型不存在或尚不可用') + driver = driver_for(model.provider, model.model_type) + if not driver: + raise ValueError('该模型由外部服务运行,不支持本地加载策略') + fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest() + return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver, + 'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)), + 'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))} + + +class WeightAdapter: + @staticmethod + def _request(spec, route, data=None): + url = urlsplit(spec['credential'].get('api_base', '')) + if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password: + raise ValueError('模型服务地址无效') + path = url.path.rstrip('/') + if path.endswith('/v1'): + path = path[:-3] + endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', '')) + key = spec['credential'].get('api_key') + headers = {'Authorization': 'Bearer ' + key} if key else {} + response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers, + timeout=(5, 600 if data is not None else 5), allow_redirects=False) + response.raise_for_status() + return response.json() + + def probe(self, spec): + try: + tags = self._request(spec, '/api/ps').get('models', []) + except requests.RequestException as exc: + raise ValueError('模型推理服务不可用') from exc + name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest' + item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None) + if item: + vram = item.get('size_vram', 0) + return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)} + + def load(self, spec): + close_old_connections() + try: + if spec['driver'] == 'ollama': + keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds'] + if spec['type'] == 'EMBEDDING': + self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive}) + else: + self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False, + 'keep_alive': keep_alive}) + info = self.probe(spec) + if not info: + raise ValueError('模型未驻留') + logger.info('Model weights ready: %s', spec['id']) + return True, info + from local_model.serializers.model_apply_serializers import get_local_model + instance = get_local_model(spec['model'], use_local=True) + if spec['type'] == 'EMBEDDING': + instance.embed_query('模型加载检查') + else: + from langchain_core.documents import Document + instance.compress_documents([Document(page_content='模型加载检查')], '模型') + client = getattr(instance, '_client', None) or getattr(instance, 'client', None) + device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu')) + info = {'device': device} + try: + import torch + if device.startswith('cuda'): + info['process_gpu_bytes'] = torch.cuda.memory_allocated(device) + except ImportError: + pass + logger.info('Model weights ready: %s', spec['id']) + return instance, info + finally: + close_old_connections() + + def unload(self, spec, value): + if spec['driver'] == 'ollama': + # Refuse to unload a native model shared by another configured resource. + from local_model.models import Model + if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists(): + raise ValueError('该模型被其他模型资源共用,不能单独卸载') + if spec['type'] == 'EMBEDDING': + self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0}) + else: + self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False}) + for _ in range(50): + if self.probe(spec) is None: + return + time.sleep(.2) + raise ValueError('模型卸载未完成,请稍后重试') + + @staticmethod + def collect(): + gc.collect() + try: + import torch + if torch.cuda.is_initialized(): + torch.cuda.empty_cache() + except ImportError: + pass + + @staticmethod + def error(exc): + name = type(exc).__name__ + logger.warning('Model weight load failed: %s', name) + if 'OutOfMemory' in name: + return '模型加载失败:显存或内存不足' + return '模型加载失败,请检查模型文件、运行设备及服务状态' + + +manager = ModelRuntime(model_spec, WeightAdapter()) +_startup_lock = threading.Lock() +_started = False + + +def start_runtime(): + global _started + with _startup_lock: + if _started: + return + _started = True + def run(): + from django.apps import apps + while not apps.ready: + time.sleep(.05) + from local_model.models import Model + try: + models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD') + ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)] + manager.preload(ids) + logger.info('Startup model preloads queued: %s', len(ids)) + except Exception as exc: + logger.warning('Startup preload discovery failed: %s', type(exc).__name__) + finally: + close_old_connections() + while True: + time.sleep(10) + try: + manager.sweep() + except Exception as exc: + logger.warning('Model idle sweep failed: %s', type(exc).__name__) + finally: + close_old_connections() + threading.Thread(target=run, name='model-startup', daemon=True).start() diff --git a/apps/local_model/views/runtime.py b/apps/local_model/views/runtime.py new file mode 100644 index 000000000..bc55fdaa4 --- /dev/null +++ b/apps/local_model/views/runtime.py @@ -0,0 +1,48 @@ +import re + +from rest_framework.exceptions import PermissionDenied, ValidationError +from rest_framework.serializers import UUIDField +from rest_framework.views import APIView + +from common.model_runtime_client import verify_request +from common.auth.authenticate import AnonymousAuthentication +from common.result import result +from local_model.runtime import manager + + +class LocalModelRuntime(APIView): + authentication_classes = [AnonymousAuthentication] + permission_classes = [] + + def post(self, request): + try: + data = verify_request(request.body, request.headers.get('X-Model-Runtime-Signature')) + except (ValueError, TypeError): + raise PermissionDenied('模型服务认证失败') + model_id = str(UUIDField().run_validation(data.get('model_id'))) + operation = data.get('operation') + try: + if operation == 'release': + lease = data.get('lease', '') + if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{32}', lease): + raise ValueError('请求标识无效') + manager.release(model_id, lease) + return result.success({'released': True}) + if operation == 'acquire': + lease = manager.acquire(model_id, remote=True) + try: + return result.success({**manager.status(model_id), 'lease': lease}) + except Exception: + manager.release(model_id, lease) + raise + if operation == 'load': + manager.submit(model_id) + elif operation == 'unload': + manager.unload(model_id) + elif operation == 'configure': + manager.configure(model_id) + elif operation != 'status': + raise ValueError('模型服务操作无效') + return result.success(manager.status(model_id)) + except (ValueError, TimeoutError) as exc: + raise ValidationError(str(exc)) diff --git a/apps/models_provider/local_imports.py b/apps/models_provider/local_imports.py new file mode 100644 index 000000000..0fa4f1204 --- /dev/null +++ b/apps/models_provider/local_imports.py @@ -0,0 +1,179 @@ +"""Model file assets, independent of runtime/model connection records. Never executes model code.""" +import json +import os +import shutil +import stat +import tempfile +import uuid +import zipfile +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + +MAX_BYTES = 2 * 1024 ** 3 +MAX_FILES = 10000 +WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'} + + +class ImportError(ValueError): + pass + + +def inspect_model(directory): + root = Path(directory) + if not root.is_dir() or root.is_symlink(): + raise ImportError('请选择有效模型目录') + files = [] + for current, dirs, names in os.walk(root, followlinks=False): + for name in dirs + names: + if (Path(current) / name).is_symlink(): + raise ImportError('模型目录不能包含符号链接') + for name in names: + file = Path(current) / name + if not file.is_file(): + raise ImportError('模型目录包含非普通文件') + files.append(file) + if len(files) > MAX_FILES: + raise ImportError('模型文件数量超过限制') + weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0] + if not weights: + raise ImportError('未找到有效模型权重文件') + + def read_json(name): + file = root / name + if not file.exists(): + return {} + if file.stat().st_size > 2 * 1024 ** 2: + raise ImportError(name + '超过大小限制') + try: + data = json.loads(file.read_text(encoding='utf-8')) + if not isinstance(data, dict): + raise ValueError() + return data + except (ValueError, UnicodeError): + raise ImportError(name + '格式错误') + + config = read_json('config.json') + manifest = read_json('model-manifest.json') + quant = config.get('quantization_config') or {} + if not isinstance(quant, dict): + raise ImportError('量化配置格式错误') + quantization = str(quant.get('quant_method', '')).lower() + suffixes = {f.suffix.lower() for f in weights} + if config and suffixes & {'.safetensors', '.bin'}: + if not config.get('model_type') and not config.get('architectures'): + raise ImportError('config.json缺少模型架构信息') + format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF') + elif '.onnx' in suffixes: + format_name = 'ONNX' + elif '.gguf' in suffixes: + format_name = 'GGUF' + elif manifest: + format_name = 'CUSTOM' + else: + raise ImportError('模型目录缺少config.json或model-manifest.json') + # Validate referenced shards without trusting paths supplied by model files. + for index in root.glob('*.index.json'): + mapping = read_json(index.name).get('weight_map') + if not isinstance(mapping, dict) or not mapping: + raise ImportError('权重索引格式错误') + for name in mapping.values(): + if not isinstance(name, str) or '\\' in name: + raise ImportError('权重分片路径无效') + relative = PurePosixPath(name) + if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file(): + raise ImportError('权重分片缺失或路径无效') + return { + 'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files), + 'file_count': len(files), 'weight_count': len(weights), + 'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config}, + 'quantization': quantization, + 'checks': ['权重文件存在', '配置JSON及分片引用检查通过'], + 'runtime_status': 'NOT_CHECKED', + 'runtime_note': '文件已导入,运行能力需接入验证', + } + + +class ModelImportStore: + def __init__(self, root): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + + def list(self): + records = [] + for path in self.root.glob('*/import.json'): + try: + records.append(json.loads(path.read_text(encoding='utf-8'))) + except (ValueError, OSError): + continue + return sorted(records, key=lambda r: r['created_at'], reverse=True)[:200] + + def create(self, owner_id, path=None, upload=None): + if (path is None) == (upload is None): + raise ImportError('请选择一种导入方式') + identity = str(uuid.uuid4()) + staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root)) + try: + if upload is not None: + if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES: + raise ImportError('请上传2GB以内的ZIP模型包') + archive = staging / 'upload.zip' + with archive.open('wb') as output: + size = 0 + for chunk in upload.chunks(): + size += len(chunk) + if size > MAX_BYTES: + raise ImportError('模型包超过2GB') + output.write(chunk) + source = staging / 'files' + source.mkdir() + try: + with zipfile.ZipFile(archive) as zipped: + members = zipped.infolist() + if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES: + raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录') + seen = set() + for member in members: + name = PurePosixPath(member.filename) + mode = member.external_attr >> 16 + if (name.is_absolute() or '..' in name.parts or '\\' in member.filename + or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1 + or member.filename in seen): + raise ImportError('模型包包含不安全的路径或文件') + seen.add(member.filename) + target = source.joinpath(*name.parts) + if member.is_dir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with zipped.open(member) as incoming, target.open('xb') as output: + shutil.copyfileobj(incoming, output, 1024 * 1024) + except (zipfile.BadZipFile, RuntimeError, OSError) as exc: + raise ImportError('模型包损坏或无法解压') from exc + archive.unlink() + # Most model archives contain one enclosing directory. + children = list(source.iterdir()) + model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source + info = inspect_model(model_dir) + final_path = self.root / identity / model_dir.relative_to(staging) + source_type = 'UPLOAD' + display_name = Path(upload.name).stem + else: + if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute(): + raise ImportError('请输入服务器绝对目录') + model_dir = Path(path).resolve(strict=True) + info = inspect_model(model_dir) + final_path, source_type, display_name = model_dir, 'PATH', model_dir.name + record = { + 'id': identity, 'name': display_name, 'path': str(final_path), + 'source': source_type, 'owner_id': str(owner_id), + 'created_at': datetime.now(timezone.utc).isoformat(), + 'status': 'IMPORTED', **info, + } + (staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8') + staging.rename(self.root / identity) + return record + except (FileNotFoundError, PermissionError) as exc: + raise ImportError('目录不存在或无访问权限') from exc + finally: + if staging.exists(): + shutil.rmtree(staging) diff --git a/apps/models_provider/migrations/0002_llm_top_p_params_form.py b/apps/models_provider/migrations/0002_llm_top_p_params_form.py new file mode 100644 index 000000000..d93ca1c38 --- /dev/null +++ b/apps/models_provider/migrations/0002_llm_top_p_params_form.py @@ -0,0 +1,28 @@ +"""补齐已有百炼模型表单中的 TopP;保留全部自定义参数项。""" +from django.db import migrations + +TOP_P_FIELD = { + "input_type": "Slider", "label": "TopP", "required": True, + "default_value": 0.8, "relation_show_field_dict": {}, + "relation_trigger_field_dict": {}, "trigger_type": "OPTION_LIST", + "attrs": {"min": 0, "max": 1, "step": 0.01, "precision": 2, + "show-input-controls": False, "show-input": True}, + "props_info": {}, "field": "top_p", +} + + +def add_top_p_params_form(apps, schema_editor): + Model = apps.get_model('models_provider', 'Model') + for model in Model.objects.using(schema_editor.connection.alias).filter( + provider='aliyun_bai_lian_model_provider', model_type='LLM'): + form = model.model_params_form or [] + if any(item.get('field') == 'top_p' for item in form): + continue + model.model_params_form = [*form, TOP_P_FIELD] + model.save(update_fields=['model_params_form'], using=schema_editor.connection.alias) + + +class Migration(migrations.Migration): + dependencies = [('models_provider', '0001_initial')] + # 回退代码时保留参数项,不删除可能由管理员维护的数据。 + operations = [migrations.RunPython(add_top_p_params_form, migrations.RunPython.noop)] diff --git a/apps/models_provider/test_local_imports.py b/apps/models_provider/test_local_imports.py new file mode 100644 index 000000000..add79d88d --- /dev/null +++ b/apps/models_provider/test_local_imports.py @@ -0,0 +1,109 @@ +"""File import regression tests, temporary files only; no database or model runtimes.""" +import io +import json +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch +from models_provider.local_imports import ModelImportStore, inspect_model, ImportError + + +class Upload: + def __init__(self, entries): + data = io.BytesIO() + with zipfile.ZipFile(data, 'w') as archive: + for name, content in entries: + archive.writestr(name, content) + self.body = data.getvalue() + self.size, self.name = len(self.body), 'model.zip' + + def chunks(self): + yield self.body + + +class ImportTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.model = self.root / 'model' + self.model.mkdir() + (self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'})) + (self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights') + self.store = ModelImportStore(self.root / 'imports') + + def test_local_path_record_persists_without_copying_or_running(self): + record = self.store.create('owner', path=str(self.model)) + self.assertEqual(record['format'], 'HF') + self.assertEqual(record['runtime_status'], 'NOT_CHECKED') + self.assertEqual(record['path'], str(self.model.resolve())) + self.assertEqual(ModelImportStore(self.store.root).list(), [record]) + + def test_quantized_formats(self): + for method in ('gptq', 'awq'): + (self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}})) + self.assertEqual(inspect_model(self.model)['format'], method.upper()) + + def test_onnx_gguf_custom(self): + (self.model / 'config.json').unlink() + (self.model / 'model.safetensors').unlink() + for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')): + file = self.model / ('model' + suffix) + file.write_bytes(b'fixture') + (self.model / 'model-manifest.json').write_text('{"version":"1"}') + self.assertEqual(inspect_model(self.model)['format'], expected) + file.unlink() + + def test_zip_nested_model_and_persisted_path(self): + record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')])) + self.assertTrue(Path(record['path'], 'model.safetensors').exists()) + self.assertEqual(record['source'], 'UPLOAD') + + def test_traversal_windows_path_rejected(self): + for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'): + with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')])) + self.assertEqual(self.store.list(), []) + + def test_duplicate_archive_entry_rejected(self): + with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')])) + + def test_zip_symlink_rejected(self): + data=io.BytesIO() + with zipfile.ZipFile(data,'w') as archive: + info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16) + archive.writestr(info, '../outside') + upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body) + with self.assertRaises(ImportError):self.store.create('owner',upload=upload) + + def test_size_limit_and_cleanup(self): + upload=Upload([('model.onnx', b'x'*100)]) + with patch('models_provider.local_imports.MAX_BYTES', 20): + with self.assertRaises(ImportError):self.store.create('owner',upload=upload) + self.assertEqual(list(self.store.root.iterdir()), []) + + def test_bad_json_missing_weights_and_unknown_folder_rejected(self): + (self.model / 'config.json').write_text('{broken') + with self.assertRaises(ImportError):inspect_model(self.model) + (self.model / 'config.json').write_text('{"other":1}') + with self.assertRaises(ImportError):inspect_model(self.model) + (self.model / 'model.safetensors').unlink() + with self.assertRaises(ImportError):inspect_model(self.model) + + def test_missing_or_escaping_shard_rejected(self): + for name in ('absent.safetensors','../outside.bin'): + (self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}})) + with self.assertRaises(ImportError):inspect_model(self.model) + + def test_bad_zip_and_missing_path_leave_no_records(self): + upload=Upload([]);upload.body=b'not zip';upload.size=7 + with self.assertRaises(ImportError):self.store.create('owner',upload=upload) + with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent')) + self.assertEqual(self.store.list(), []) + + def test_exclusive_source(self): + for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}): + with self.assertRaises(ImportError):self.store.create('owner',**kwargs) + + +if __name__ == '__main__':unittest.main() diff --git a/apps/models_provider/test_model_loading_api.py b/apps/models_provider/test_model_loading_api.py new file mode 100644 index 000000000..673699e81 --- /dev/null +++ b/apps/models_provider/test_model_loading_api.py @@ -0,0 +1,67 @@ +import json +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import TestCase +from rest_framework.exceptions import ValidationError + +from common.constants.permission_constants import RoleConstants +from common.exception.app_exception import AppUnauthorizedFailed +from models_provider.models import Model +from models_provider.views.runtime import ModelLoading + + +class LoadingApiTests(TestCase): + def setUp(self): + self.model = Model.objects.create(name='loading-test', provider='model_local_provider', + model_type='EMBEDDING', model_name='test-only', credential='test-only', meta={'existing': 'keep'}) + self.admin = SimpleNamespace(auth=SimpleNamespace(role_list=[str(RoleConstants.ADMIN.value)], permission_list=[]), data={}) + self.view = ModelLoading() + + def args(self): + return {'workspace_id': 'default', 'model_id': str(self.model.id)} + + def test_policy_is_persistent_and_preserves_existing_metadata(self): + self.admin.data = {'strategy': 'PRELOAD', 'priority': 90} + with patch('models_provider.views.runtime.runtime_request', return_value={'status': 'QUEUED'}) as rpc: + response = self.view.put(self.admin, **self.args()) + self.model.refresh_from_db() + self.assertEqual(self.model.meta['existing'], 'keep') + self.assertEqual(self.model.meta['loading_policy'], {'strategy': 'PRELOAD', 'priority': 90, 'idle_seconds': 300}) + self.assertEqual(json.loads(response.content)['data']['status'], 'QUEUED') + self.assertEqual(rpc.call_args_list[0].args[0], 'configure') + + def test_normal_user_cannot_change_policy_or_load_weights(self): + user = SimpleNamespace(auth=SimpleNamespace(role_list=[], permission_list=[]), data={'action': 'load'}) + for method in (self.view.get, self.view.put, self.view.post): + with self.assertRaises(AppUnauthorizedFailed): + method(user, **self.args()) + + def test_workspace_and_unsupported_provider_are_rejected(self): + with self.assertRaises(ValidationError): + self.view.get(self.admin, workspace_id='other', model_id=str(self.model.id)) + self.model.provider = 'aliyun_bai_lian_model_provider' + self.model.model_type = 'LLM'; self.model.save() + response = self.view.get(self.admin, **self.args()) + self.assertEqual(json.loads(response.content)['data']['status'], 'EXTERNAL') + with self.assertRaises(ValidationError): + self.view.put(self.admin, **self.args()) + + def test_arbitrary_actions_and_invalid_policies_do_not_reach_runtime(self): + with patch('models_provider.views.runtime.runtime_request') as rpc: + self.admin.data = {'action': 'shell'} + with self.assertRaises(ValidationError): self.view.post(self.admin, **self.args()) + self.admin.data = {'priority': 500} + with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args()) + rpc.assert_not_called() + self.model.refresh_from_db() + self.assertNotIn('loading_policy', self.model.meta) + + def test_saved_policy_remains_available_when_runtime_is_temporarily_down(self): + self.admin.data = {'strategy': 'PRELOAD', 'priority': 80} + with patch('models_provider.views.runtime.runtime_request', side_effect=ValueError('offline')): + with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args()) + response = self.view.get(self.admin, **self.args()) + self.model.refresh_from_db() + self.assertEqual(self.model.meta['loading_policy']['priority'], 80) + self.assertEqual(json.loads(response.content)['data']['status'], 'UNAVAILABLE') diff --git a/apps/models_provider/test_model_runtime.py b/apps/models_provider/test_model_runtime.py new file mode 100644 index 000000000..3396fe94c --- /dev/null +++ b/apps/models_provider/test_model_runtime.py @@ -0,0 +1,199 @@ +import contextlib +import json +import threading +import weakref +import unittest +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from unittest.mock import patch + +from common.model_runtime import ModelRuntime, loading_policy +from common.model_runtime_client import signed_request, verify_request + + +class FakeAdapter: + def __init__(self): + self.loaded, self.unloaded = [], [] + self.native_ready = True + self.fail = False + self.block = None + + def load(self, spec): + if self.block: + self.block.wait(3) + if self.fail: + raise ValueError('private error details') + self.loaded.append(spec['id']) + self.native_ready = True + return SimpleNamespace(), {'device': 'cpu'} + + def unload(self, spec, value): + self.unloaded.append(spec['id']) + self.native_ready = False + + def probe(self, spec): + return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None + + def collect(self): + pass + + def error(self, exc): + return 'load failed' + + +class RuntimeTests(unittest.TestCase): + def setUp(self): + self.now = 0 + self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1', + 'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}} + for key, priority in [('low', 10), ('high', 90)]} + self.adapter = FakeAdapter() + self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])}, + self.adapter, lambda: self.now) + + def test_no_load_until_first_inference_and_then_reuses_weights(self): + self.assertEqual(self.manager.status('low')['status'], 'UNLOADED') + self.assertEqual(self.adapter.loaded, []) + with self.manager.inference('low') as first: + self.assertIsNotNone(first) + with self.manager.inference('low') as second: + self.assertIs(first, second) + self.assertEqual(self.adapter.loaded, ['low']) + + def test_startup_preloads_highest_priority_first(self): + self.specs['low']['policy']['strategy'] = 'PRELOAD' + self.specs['high']['policy']['strategy'] = 'PRELOAD' + self.manager.preload(['low', 'high']) + for key in ['low', 'high']: + self.manager.entries[key]['future'].result(3) + self.assertEqual(self.adapter.loaded, ['high', 'low']) + self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order']) + + def test_concurrent_first_calls_only_load_once(self): + self.adapter.block = threading.Event() + with ThreadPoolExecutor(max_workers=6) as pool: + futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)] + self.adapter.block.set() + tokens = [future.result(3) for future in futures] + self.assertEqual(len(set(tokens)), 6) + self.assertEqual(self.adapter.loaded, ['low']) + self.assertEqual(self.manager.status('low')['active_requests'], 6) + for token in tokens: + self.manager.release('low', token) + + def test_in_use_models_cannot_be_unloaded(self): + token = self.manager.acquire('low') + with self.assertRaises(ValueError): + self.manager.unload('low') + self.manager.release('low', token) + self.manager.unload('low') + self.assertEqual(self.manager.status('low')['status'], 'UNLOADED') + + def test_unload_releases_the_last_loader_reference(self): + class Weights: + pass + with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})): + token = self.manager.acquire('low') + pointer = weakref.ref(self.manager.entries['low']['value']) + self.manager.release('low', token) + self.manager.unload('low') + self.assertIsNone(pointer()) + + def test_idle_cleanup_respects_active_and_preloaded_models(self): + active = self.manager.acquire('low') + self.specs['high']['policy']['strategy'] = 'PRELOAD' + self.manager.preload(['high']) + self.manager.entries['high']['future'].result(3) + self.now = 100 + self.manager.sweep() + self.assertEqual(self.adapter.unloaded, []) + self.manager.release('low', active) + self.now = 131 + self.manager.sweep() + self.assertEqual(self.adapter.unloaded, ['low']) + self.assertEqual(self.manager.status('high')['status'], 'READY') + + def test_failed_load_can_be_retried_and_private_errors_are_hidden(self): + self.adapter.fail = True + with self.assertRaisesRegex(ValueError, '^load failed$'): + self.manager.acquire('low') + self.assertEqual(self.manager.status('low')['status'], 'FAILED') + self.adapter.fail = False + with self.manager.inference('low'): + pass + self.assertEqual(self.manager.status('low')['status'], 'READY') + + def test_changed_model_config_never_reuses_old_weights(self): + token = self.manager.acquire('low') + self.specs['low']['fingerprint'] = 'v2' + with self.assertRaises(ValueError): + self.manager.acquire('low') + self.manager.release('low', token) + with self.manager.inference('low'): + pass + self.assertEqual(self.adapter.loaded, ['low', 'low']) + self.assertEqual(self.adapter.unloaded, ['low']) + + def test_native_expiration_requires_loading_again(self): + self.specs['low']['driver'] = 'ollama' + token = self.manager.acquire('low') + self.manager.release('low', token) + self.adapter.native_ready = False + token = self.manager.acquire('low') + self.manager.release('low', token) + self.assertEqual(self.adapter.loaded, ['low', 'low']) + + def test_policy_validation(self): + for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'}, + {'idle_seconds': 0}, {'engine_url': 'http://example.com'}): + with self.subTest(value=value), self.assertRaises(ValueError): + loading_policy(value) + + def test_internal_rpc_rejects_forged_or_expired_payloads(self): + with patch('common.model_runtime_client._secret', return_value=b'test-only-key'): + body, signature = signed_request({'operation': 'load', 'model_id': 'test'}) + self.assertEqual(verify_request(body, signature)['operation'], 'load') + with self.assertRaises(ValueError): + verify_request(body.replace(b'load', b'unload'), signature) + with self.assertRaises(ValueError): + verify_request(body, '') + with patch('common.model_runtime_client.time.time', return_value=1): + body, signature = signed_request({'operation': 'load'}) + with self.assertRaises(ValueError): + verify_request(body, signature) + + +class ProviderHookTests(unittest.TestCase): + def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self): + from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding + events = [] + @contextlib.contextmanager + def lease(model_id): + events.append(('acquire', model_id)) + try: + yield {'strategy': 'PRELOAD', 'idle_seconds': 300} + finally: + events.append(('release', model_id)) + model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource') + with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]): + self.assertEqual(model.embed_query('text'), [1., 2.]) + self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')]) + self.assertEqual(model._default_params['keep_alive'], -1) + + def test_llm_stream_releases_lease_when_generator_is_closed(self): + from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel + from langchain_ollama import ChatOllama + events = [] + @contextlib.contextmanager + def lease(model_id): + events.append('acquire') + try: + yield {'strategy': 'PRELOAD', 'idle_seconds': 300} + finally: + events.append('release') + model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource') + with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native: + stream = model._create_chat_stream([]) + next(stream); stream.close() + self.assertEqual(native.call_args.kwargs['keep_alive'], -1) + self.assertEqual(events, ['acquire', 'release']) diff --git a/apps/models_provider/views/local_imports.py b/apps/models_provider/views/local_imports.py new file mode 100644 index 000000000..e6088f99e --- /dev/null +++ b/apps/models_provider/views/local_imports.py @@ -0,0 +1,31 @@ +import os +from pathlib import Path + +from rest_framework.views import APIView +from common.auth import TokenAuth +from common.auth.authentication import has_permissions +from common.constants.permission_constants import RoleConstants +from common.exception.app_exception import AppApiException +from common.result import result +from maxkb.conf import PROJECT_DIR +from models_provider.local_imports import ModelImportStore, ImportError + + +class LocalModelImports(APIView): + authentication_classes = [TokenAuth] + + @staticmethod + def store(): + return ModelImportStore(os.environ.get('MAXKB_MODEL_IMPORT_DIR', str(Path(PROJECT_DIR) / 'model' / 'imports'))) + + @has_permissions(RoleConstants.ADMIN) + def get(self, request): + return result.success(self.store().list()) + + @has_permissions(RoleConstants.ADMIN) + def post(self, request): + try: + return result.success(self.store().create(request.user.id, + path=request.data.get('path'), upload=request.FILES.get('file'))) + except ImportError as exc: + raise AppApiException(500, str(exc)) diff --git a/apps/models_provider/views/runtime.py b/apps/models_provider/views/runtime.py new file mode 100644 index 000000000..896ee23fa --- /dev/null +++ b/apps/models_provider/views/runtime.py @@ -0,0 +1,75 @@ +from django.db import transaction +from rest_framework.exceptions import ValidationError +from rest_framework.serializers import UUIDField +from rest_framework.views import APIView + +from common.auth import TokenAuth +from common.auth.authentication import has_permissions +from common.constants.permission_constants import RoleConstants +from common.model_runtime import loading_policy +from common.model_runtime_client import runtime_request +from common.result import result +from models_provider.models import Model + + +def managed_model(model): + return ((model.provider == 'model_local_provider' and model.model_type in ('EMBEDDING', 'RERANKER')) or + (model.provider == 'model_ollama_provider' and model.model_type in ('EMBEDDING', 'LLM'))) + + +class ModelLoading(APIView): + authentication_classes = [TokenAuth] + + @staticmethod + def model(workspace_id, model_id): + model_id = UUIDField().run_validation(model_id) + model = Model.objects.filter(id=model_id, workspace_id=workspace_id).first() + if model is None: + raise ValidationError('模型不存在') + return model + + @staticmethod + def state(model): + policy = loading_policy((model.meta or {}).get('loading_policy')) + if not managed_model(model): + return {**policy, 'supported': False, 'status': 'EXTERNAL'} + try: + return {**runtime_request('status', model.id), 'supported': True} + except ValueError: + return {**policy, 'supported': True, 'status': 'UNAVAILABLE', 'error': '本地模型服务不可用'} + + @has_permissions(RoleConstants.ADMIN) + def get(self, request, workspace_id, model_id): + return result.success(self.state(self.model(workspace_id, model_id))) + + @has_permissions(RoleConstants.ADMIN) + def put(self, request, workspace_id, model_id): + model = self.model(workspace_id, model_id) + if not managed_model(model): + raise ValidationError('该模型由外部服务运行') + try: + policy = loading_policy(request.data) + except ValueError as exc: + raise ValidationError(str(exc)) + with transaction.atomic(): + model = Model.objects.select_for_update().get(id=model.id) + model.meta = {**(model.meta or {}), 'loading_policy': policy} + model.save(update_fields=['meta', 'update_time']) + try: + runtime_request('configure', model.id) + except ValueError: + raise ValidationError('策略已保存,本地模型服务暂不可用,请恢复服务后重试') + return result.success(self.state(model)) + + @has_permissions(RoleConstants.ADMIN) + def post(self, request, workspace_id, model_id): + model = self.model(workspace_id, model_id) + if not managed_model(model): + raise ValidationError('该模型由外部服务运行') + operation = request.data.get('action') + if operation not in ('load', 'unload'): + raise ValidationError('无效操作') + try: + return result.success({**runtime_request(operation, model.id), 'supported': True}) + except ValueError as exc: + raise ValidationError(str(exc)) diff --git a/apps/system_manage/ai_hub.py b/apps/system_manage/ai_hub.py new file mode 100644 index 000000000..5047c9e9f --- /dev/null +++ b/apps/system_manage/ai_hub.py @@ -0,0 +1,453 @@ +"""AI hub business operations. Secrets never cross the browser boundary.""" +from urllib.parse import urlsplit +from django.db import transaction +from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound +from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact + +MAX_TEXT = 2_000_000 +MAX_INLINE_TEXT = MAX_TEXT + 140_000_000 +MAX_FILE = 20 * 1024 * 1024 +KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService, + 'tasks': AITask, 'artifacts': AIArtifact} + + +def admin(user): + from common.constants.permission_constants import RoleConstants + return user.role == str(RoleConstants.ADMIN.value) + + +def require_admin(user): + if not admin(user): + raise PermissionDenied('仅管理员可操作') + + +def visible(query, user): + # Sharing is explicit, with no implicit public access. + if admin(user): + return query + if query.model in (AIParserResource, AIService): + ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id) + or str(user.id) in obj.allowed_users] + return query.filter(id__in=ids) + return query.filter(owner_id=user.id) + + +def one(kind, pk, workspace, user, edit=False): + from rest_framework.serializers import UUIDField + pk = UUIDField().run_validation(pk) + model = KINDS.get(kind) + if not model: + raise NotFound() + obj = visible(model.objects.filter(workspace_id=workspace), user).filter(id=pk).first() + if not obj: + raise NotFound('记录不存在或无权访问') + if edit and not admin(user) and str(obj.owner_id) != str(user.id): + raise PermissionDenied('无权修改') + return obj + + +def text(value, label, required=True, limit=MAX_TEXT): + if not isinstance(value, str) or len(value) > limit or (required and not value.strip()): + raise ValidationError(f'{label}无效') + return value + + +def integer(value, minimum, maximum, label): + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: + raise ValidationError(f'{label}无效') + return value + + +def users(value): + from users.models import User + if not isinstance(value, list) or len(value) > 500: + raise ValidationError('授权用户无效') + values = {str(item) for item in value} + try: + allowed = set(str(pk) for pk in User.objects.filter(id__in=values, is_active=True).values_list('id', flat=True)) + except (ValueError, TypeError): + raise ValidationError('授权用户无效') + # Registration approval is enforced by the existing activation extension. + from users.models.user_manage_ext import UserActivationApplication + pending = set(str(pk) for pk in UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)) + if values - (allowed - pending): + raise ValidationError('只能授权已激活且启用的用户') + return sorted(values) + + +def authorized_models(workspace, user): + from models_provider.models import Model + from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType + query = Model.objects.filter(workspace_id=workspace, status='SUCCESS') + if not admin(user): + grants = WorkspaceUserResourcePermission.objects.filter( + workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL) + ids = [g.target for g in grants if 'VIEW' in g.permission_list] + query = query.filter(id__in=ids) + return query + + +def model_for(workspace, user, model_id): + from rest_framework.serializers import UUIDField + model_id = UUIDField().run_validation(model_id) + obj = authorized_models(workspace, user).filter(id=model_id, model_type='LLM').first() + if not obj: + raise PermissionDenied('模型不可用或未授权') + return obj + + +def serialize(obj, detail=False): + data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()} + if isinstance(obj, AIParserResource): + data.update(enabled=obj.enabled, allowed_users=obj.allowed_users, + options=obj.options, credential_configured=bool(obj.credential)) + # Only callers of admin management endpoints receive endpoint addresses. + elif isinstance(obj, AIFlow): + data.update(revision=obj.revision, nodes=obj.nodes) + elif isinstance(obj, AIService): + data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users, + requests_per_minute=obj.requests_per_minute, + key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run') + elif isinstance(obj, AITask): + from system_manage.ai_hub_progress import task_progress + data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt, + started_at=obj.started_at, finished_at=obj.finished_at) + if detail: + data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj)) + elif isinstance(obj, AIArtifact): + data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None) + if detail: + data.update(original=obj.original, markdown=obj.markdown, history=obj.history) + return data + + +def parser_save(workspace, user, data, obj=None): + require_admin(user) + endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/') + parsed = urlsplit(endpoint) + if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValidationError('解析服务地址无效') + obj = obj or AIParserResource(workspace_id=workspace, owner_id=user.id) + obj.name = text(data.get('name', obj.name), '名称', limit=128) + obj.endpoint = endpoint + if 'api_key' in data: + from common.utils.rsa_util import rsa_long_encrypt + key = text(data['api_key'], '密钥', required=False, limit=4096) + obj.credential = rsa_long_encrypt(key) if key else '' + enabled = data.get('enabled', obj.enabled) + if not isinstance(enabled, bool): + raise ValidationError('启用状态无效') + obj.enabled = enabled + if 'allowed_users' in data: + obj.allowed_users = users(data['allowed_users']) + options = data.get('options', obj.options) + if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier'}: + raise ValidationError('解析参数无效') + if options.get('protocol', 'legacy') not in ('legacy', 'v1') or options.get('tier', 'standard') not in ('flash', 'basic', 'standard', 'advanced'): + raise ValidationError('解析接口或档位无效') + if options.get('backend', 'pipeline') not in ('pipeline', 'hybrid-auto-engine', 'vlm-auto-engine'): + raise ValidationError('不支持的解析后端') + if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'): + raise ValidationError('识别语言无效') + if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options): + raise ValidationError('解析参数无效') + obj.options = options + obj.save() + return obj + + +def parser_test(parser): + """Probe the registered service API, without submitting files or returning service content.""" + import time + import requests + from urllib.parse import urlunsplit + from common.utils.rsa_util import rsa_long_decrypt + parsed = urlsplit(parser.endpoint.rstrip('/')) + if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password: + raise ValidationError('解析服务地址无效') + base_path = parsed.path + for suffix in ('/v1', '/api'): + if base_path.endswith(suffix): + base_path = base_path[:-len(suffix)] + break + url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', '')) + headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} + started = time.monotonic() + try: + response = requests.get(url, headers=headers, timeout=(3, 7), allow_redirects=False) + if response.status_code != 200: + raise ValidationError(f'连接测试失败(HTTP {response.status_code})') + data = response.json() + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + except ValueError as exc: + raise ValidationError('解析服务未返回有效接口定义') from exc + paths = data.get('paths') if isinstance(data, dict) else None + v1 = parser.options.get('protocol') == 'v1' or parsed.path.endswith('/v1') + suffix = '/tasks' if v1 else '/file_parse' + if not isinstance(paths, dict) or not any(path.endswith(suffix) and 'post' in methods + for path, methods in paths.items() if isinstance(methods, dict)): + raise ValidationError('未识别到对应的解析接口') + return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000), + 'message': '解析接口连接正常'} + + +def validate_nodes(workspace, user, nodes): + if not isinstance(nodes, list) or not 1 <= len(nodes) <= 3: + raise ValidationError('请选择处理节点') + types = [n.get('type') for n in nodes if isinstance(n, dict)] + if len(types) != len(nodes) or types[-1] != 'render' or len(set(types)) != len(types): + raise ValidationError('流程须以导图生成为终点,节点不可重复') + if types != [t for t in ('parse', 'organize', 'render') if t in types]: + raise ValidationError('节点顺序应为解析、整理、导图生成') + result = [] + for node in nodes: + kind = node['type'] + clean = {'type': kind} + if kind == 'parse': + parser = one('parsers', node.get('resource_id'), workspace, user) + if not parser.enabled: + raise ValidationError('解析服务已停用') + clean['resource_id'] = str(parser.id) + if kind == 'organize': + clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id) + clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000) + temperature = node.get('temperature', 0.2) + if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2: + raise ValidationError('温度无效') + clean['temperature'] = temperature + result.append(clean) + return result + + +@transaction.atomic +def flow_create(workspace, user, data): + nodes = validate_nodes(workspace, user, data.get('nodes')) + # Every save is immutable; published services always reference a fixed row. + previous = one('flows', data['previous_id'], workspace, user, edit=True) if data.get('previous_id') else None + return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id, + name=text(data.get('name'), '名称', limit=128), nodes=nodes, + revision=previous.revision + 1 if previous else 1) + + +@transaction.atomic +def service_create(workspace, user, data): + require_admin(user) + flow = one('flows', data.get('flow_id'), workspace, user) + validate_nodes(workspace, user, flow.nodes) + return AIService.objects.create(workspace_id=workspace, owner_id=user.id, + name=text(data.get('name'), '名称', limit=128), flow=flow, + allowed_users=users(data.get('allowed_users', [])), + requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率')) + + +def make_snapshot(workspace, user, nodes): + clean = validate_nodes(workspace, user, nodes) + for node in clean: + if node['type'] == 'parse': + parser = one('parsers', node['resource_id'], workspace, user) + node['options'] = dict(parser.options) + node['resource_version'] = parser.updated_at.isoformat() + if node['type'] == 'organize': + model = model_for(workspace, user, node['model_id']) + node.update(model_name=model.model_name, provider=model.provider, resource_version=model.update_time.isoformat()) + return clean + + +@transaction.atomic +def submit(workspace, user, data, upload=None, service=None): + from users.models import User + current_user = User.objects.select_for_update().get(id=user.id) + if not current_user.is_active: + raise PermissionDenied('用户已停用') + if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10: + raise ValidationError('待处理任务已达上限') + if service: + if not service.enabled: + raise ValidationError('服务已停用') + nodes = service.flow.nodes + else: + nodes = one('flows', data['flow_id'], workspace, user).nodes if data.get('flow_id') else data.get('nodes') + snapshot = make_snapshot(workspace, user, nodes) + markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT) + if upload and upload.size > MAX_FILE: + raise ValidationError('文件不能超过20MB') + if upload and snapshot[0]['type'] != 'parse': + raise ValidationError('文件任务需要解析节点') + if not upload and snapshot[0]['type'] == 'parse': + raise ValidationError('请选择文件') + if not upload and not markdown.strip(): + raise ValidationError('文档内容不能为空') + from system_manage.ai_hub_assets import asset_lock, normalize + with asset_lock(): + markdown = text(normalize(markdown, workspace, user), '文档内容', required=False) + return AITask.objects.create(workspace_id=workspace, owner_id=user.id, + name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service, + snapshot={'nodes': snapshot, 'flow_id': str(service.flow_id) if service else data.get('flow_id'), + 'flow_revision': service.flow.revision if service else None}, + input_data={'markdown': markdown}, file_name=upload.name[:255] if upload else '', + file_data=upload.read() if upload else None) + + +@transaction.atomic +def artifact_save(workspace, user, data, obj=None): + if obj: + obj = AIArtifact.objects.select_for_update().get(pk=obj.pk) + if data.get('revision') != obj.revision: + raise ValidationError('内容已更新,请重新打开') + history = list(obj.history) + if len(history) >= 30: + raise ValidationError('修订已达30次,请另存为新图谱') + history.append({'revision': obj.revision, 'markdown': obj.markdown, 'layout': obj.layout, + 'operator': str(user.id), 'updated_at': obj.updated_at.isoformat()}) + obj.history = history + obj.revision += 1 + else: + task = one('tasks', data['task_id'], workspace, user) if data.get('task_id') else None + if task and task.status != 'SUCCESS': + raise ValidationError('任务尚未成功') + obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task, + original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT)) + obj.name = text(data.get('name'), '名称', limit=128) + obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT) + obj.layout = data.get('layout', 'tree') + if obj.layout not in ('tree', 'radial', 'horizontal', 'vertical'): + raise ValidationError('布局无效') + from system_manage.ai_hub_assets import asset_lock, normalize + with asset_lock(): + obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容') + obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False) + for entry in obj.history: + entry['markdown'] = normalize(entry.get('markdown', ''), workspace, user) + obj.save() + return obj + + +def parse_file(parser, name, content, options=None, progress_callback=None): + import requests + from common.utils.rsa_util import rsa_long_decrypt + opts = options if options is not None else parser.options + headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {} + if opts.get('protocol') == 'v1' or parser.endpoint.rstrip('/').endswith('/v1'): + from system_manage.ai_hub_parser_v1 import parse + return parse(parser, name, content, opts, headers, progress_callback=progress_callback) + try: + response = requests.post(parser.endpoint + '/file_parse', files={'files': (name, content)}, headers=headers, + data={'backend': opts.get('backend', 'pipeline'), 'lang_list': opts.get('language', 'ch'), + 'parse_method': 'ocr' if opts.get('ocr') else 'auto', 'return_md': 'true', + 'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true', + 'formula_enable': str(opts.get('formula', True)).lower(), + 'table_enable': str(opts.get('table', True)).lower()}, timeout=(10, 300), allow_redirects=False) + except Exception as exc: + raise ValidationError('解析服务连接失败或超时') from exc + if not 200 <= response.status_code < 300: + raise ValidationError('解析服务调用失败') + try: + payload = response.json() + except ValueError as exc: + raise ValidationError('解析服务返回格式无效') from exc + if not isinstance(payload, dict): + raise ValidationError('解析服务返回格式无效') + entries = payload.get('results', {}) + if not isinstance(entries, dict) or len(entries) != 1: + raise ValidationError('解析结果格式无效') + result = next(iter(entries.values())) + if not isinstance(result, dict): + raise ValidationError('解析结果格式无效') + markdown = text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT) + return {'markdown': markdown, 'source': result.get('middle_json', {}), 'parser_id': str(parser.id), + 'images': result.get('images', {})} + + +def markdown_tree(markdown): + """Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。""" + import re + root = {'id': 'root', 'text': '文档', 'children': []} + heading_stack = [(0, root)] + list_stack = [] + fence = None + table_buffer, table_start = [], None + + def container(): + return list_stack[-1][1] if list_stack else heading_stack[-1][1] + + def flush_table(): + nonlocal table_buffer, table_start + if not table_buffer: + return + index, lines = table_start, table_buffer + table_buffer, table_start = [], None + + def cells(raw): + return [cell.strip() for cell in raw.strip().strip('|').split('|')] + + if len(lines) >= 2 and re.match(r'^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$', lines[1]): + headers = cells(lines[0]) + aligns = [] + for cell in cells(lines[1]): + left, right = cell.startswith(':'), cell.endswith(':') + aligns.append('center' if left and right else 'right' if right else 'left') + rows = [cells(raw) for raw in lines[2:]] + node = {'id': f'n{index}', 'type': 'table', 'text': '表格:' + '、'.join(headers), + 'table': {'headers': headers, 'aligns': aligns, 'rows': rows, + 'row_count': len(rows), 'column_count': len(headers)}, + 'children': []} + container()['children'].append(node) + else: + for offset, raw in enumerate(lines): + container()['children'].append({'id': f'n{index + offset}', 'text': raw.strip(), 'children': []}) + + list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$') + for index, line in enumerate(markdown.splitlines()): + fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line) + if fence_match: + token = fence_match.group(1) + if fence is None: + fence = token + flush_table() + list_stack.clear() + elif token[0] == fence[0] and len(token) >= len(fence): + fence = None + continue + if fence: + continue + stripped = line.strip() + if stripped.startswith('|') and stripped.count('|') >= 2: + if table_start is None: + table_start = index + list_stack.clear() + table_buffer.append(line) + continue + flush_table() + if not stripped: + continue + heading = re.match(r'^\s{0,3}(#{1,6})\s+(.+)', line) + if heading: + level = len(heading.group(1)) + list_stack.clear() + while heading_stack[-1][0] >= level: + heading_stack.pop() + node = {'id': f'n{index}', 'text': heading.group(2).rstrip('# ').strip(), 'children': []} + heading_stack[-1][1]['children'].append(node) + heading_stack.append((level, node)) + continue + item = list_pattern.match(line) + if item: + indent = len(item.group(1).expandtabs(4)) + while list_stack and list_stack[-1][0] >= indent: + list_stack.pop() + ordered = item.group(3) + node = {'id': f'n{index}', 'type': 'list-item', + 'text': item.group(4).strip(), + 'list': {'type': 'ol' if ordered else 'ul', 'index': int(ordered) if ordered else None}, + 'children': []} + container()['children'].append(node) + list_stack.append((indent, node)) + continue + if line[:1].isspace() and list_stack: + list_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []}) + continue + list_stack.clear() + heading_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []}) + flush_table() + return root diff --git a/apps/system_manage/ai_hub_assets.py b/apps/system_manage/ai_hub_assets.py new file mode 100644 index 000000000..873e2d8e1 --- /dev/null +++ b/apps/system_manage/ai_hub_assets.py @@ -0,0 +1,213 @@ +"""Private, durable image files. References live in existing task/artifact records. + +AI_HUB_ASSET_DIR must be on persistent shared storage when Web/worker are on +different hosts. No migration, public static route, or Redis image payloads. +""" +import base64 +import hashlib +import io +import json +import os +from pathlib import Path +import re +import time +import uuid +from contextlib import contextmanager +from urllib.parse import quote + +from django.conf import settings +from rest_framework.exceptions import NotFound, ValidationError + +DATA_IMAGE = re.compile(r'data:image/(?:png|jpe?g|gif|webp|bmp);base64,[A-Za-z0-9+/=\r\n]+', re.I) +ASSET = re.compile(r'/admin/api/workspace/([^/\s"<>]+)/ai_hub/assets/([a-f0-9-]{36})/content') +MAX_IMAGE = 20 * 1024 * 1024 +MAX_IMAGES_BYTES = 100 * 1024 * 1024 +TTL = 7 * 86400 + + +def root(): + return Path(os.environ.get('AI_HUB_ASSET_DIR') or getattr(settings, 'AI_HUB_ASSET_DIR', '') + or Path(__file__).resolve().parents[2] / 'data' / 'ai-hub-assets').resolve() + + +@contextmanager +def asset_lock(): + directory = root() + directory.mkdir(parents=True, exist_ok=True) + lock = directory / '.lock' + stream = lock.open('a+b') + stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0) + deadline = time.monotonic() + 30 + while True: + try: + if os.name == 'nt': + import msvcrt + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except OSError: + if time.monotonic() >= deadline: + stream.close() + raise ValidationError('图片存储繁忙,请稍后重试') + time.sleep(.05) + try: + yield + finally: + if os.name == 'nt': + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + stream.close() + + +def _write(path, content): + temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp') + try: + with temporary.open('wb') as stream: + stream.write(content) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _metadata(asset_id): + try: + asset_id = str(uuid.UUID(str(asset_id))) + return json.loads((root() / (asset_id + '.json')).read_text(encoding='utf-8')) + except (OSError, ValueError, TypeError): + raise NotFound('图片不存在或已过期') + + +def authorize(workspace, user, asset_id): + from system_manage.ai_hub import admin + meta = _metadata(asset_id) + if meta['workspace_id'] != workspace or (meta['owner_id'] != str(user.id) and not admin(user)): + raise NotFound('图片不存在或无权访问') + return meta + + +def image_bytes(workspace, user, asset_id): + meta = authorize(workspace, user, asset_id) + try: + content = (root() / (str(uuid.UUID(str(asset_id))) + '.bin')).read_bytes() + except OSError: + raise NotFound('图片不存在或已过期') + return content, meta + + +def _store(uri, workspace, user): + from PIL import Image, UnidentifiedImageError + try: + encoded = uri.split(',', 1)[1] + if len(encoded) > MAX_IMAGE * 4 // 3 + 16: + raise ValueError() + content = base64.b64decode(re.sub(r'\s', '', encoded), validate=True) + if not content or len(content) > MAX_IMAGE: + raise ValueError() + with Image.open(io.BytesIO(content)) as image: + if image.width * image.height > 40_000_000: + raise ValueError() + fmt = image.format + image.verify() + mime = {'PNG': 'image/png', 'JPEG': 'image/jpeg', 'GIF': 'image/gif', + 'WEBP': 'image/webp', 'BMP': 'image/bmp'}[fmt] + except (ValueError, KeyError, OSError, UnidentifiedImageError, Image.DecompressionBombError): + raise ValidationError('图片格式无效或超过大小限制') + digest = hashlib.sha256(content).hexdigest() + asset_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f'ai-hub:{workspace}:{user.id}:{digest}')) + meta = {'id': asset_id, 'workspace_id': workspace, 'owner_id': str(user.id), + 'sha256': digest, 'mime': mime, 'size': len(content), 'touched_at': time.time()} + if not (root() / (asset_id + '.bin')).exists(): + _write(root() / (asset_id + '.bin'), content) + _write(root() / (asset_id + '.json'), json.dumps(meta).encode()) + return f'/admin/api/workspace/{quote(workspace, safe="")}/ai_hub/assets/{asset_id}/content' + + +def normalize(markdown, workspace, user, images=None): + """Caller holds asset_lock until its database reference is saved.""" + replacements = {} + total = 0 + + def store(uri): + nonlocal total + if uri not in replacements: + total += len(uri) * 3 // 4 + if total > MAX_IMAGES_BYTES: + raise ValidationError('文档图片总大小超过限制') + replacements[uri] = _store(uri, workspace, user) + return replacements[uri] + + if images: + if not isinstance(images, dict): + raise ValidationError('解析图片格式无效') + # Only replace destinations in Markdown/HTML, never arbitrary words. + for name, uri in images.items(): + if not isinstance(name, str) or not isinstance(uri, str) or not DATA_IMAGE.fullmatch(uri): + continue + pattern = re.compile(r'(?P\]\("\x27])', re.M) + if pattern.search(markdown): + url = store(uri) + markdown = pattern.sub(lambda m: m['prefix'] + url, markdown) + markdown = DATA_IMAGE.sub(lambda m: store(m[0]), markdown) + for match in ASSET.finditer(markdown): + if match[1] != quote(workspace, safe=''): + raise ValidationError('不能引用其他工作空间的图片') + meta = authorize(workspace, user, match[2]) + meta['touched_at'] = time.time() + _write(root() / (match[2] + '.json'), json.dumps(meta).encode()) + return markdown + + +def protect_images(markdown): + """Keep image destinations out of model rewriting; reject lost/forged tokens.""" + images = [] + pattern = re.compile(r'!\[[^\]\n]*\]\([^\n]*?\)|]*>', re.I) + def replace(match): + images.append(match[0]) + return f'AIHUBIMAGE{len(images):04d}TOKEN' + return pattern.sub(replace, markdown), images + + +def restore_images(markdown, images): + # Summarizers may omit or repeat placeholders. The backend owns the image + # manifest, so preserve known positions and append omitted originals once. + known = {str(index + 1): image for index, image in enumerate(images)} + restored = set() + def replace(match): + index = match[1].lstrip('0') or '0' + if index not in known or index in restored: + return '' + restored.add(index) + return known[index] + markdown = re.sub(r'AIHUBIMAGE[\s_*`-]*(\d+)[\s_*`-]*TOKEN', replace, markdown, flags=re.I) + missing = [image for index, image in known.items() if index not in restored] + if missing: + markdown = markdown.rstrip() + '\n\n## 原文图片\n\n' + '\n\n'.join(missing) + return markdown + + +def cleanup(apply=False, now=None): + """Only expired, unreferenced Hub images; saved originals/history count as refs.""" + from system_manage.models.ai_hub import AIArtifact, AITask + cutoff = (now or time.time()) - TTL + expired = [] + with asset_lock(): + protected = set() + for artifact in AIArtifact.objects.only('markdown', 'original', 'history').iterator(): + content = artifact.markdown + '\n' + artifact.original + '\n' + json.dumps(artifact.history) + protected.update(m[2] for m in ASSET.finditer(content)) + for task in AITask.objects.filter(status__in=['PENDING', 'RUNNING']).only('input_data', 'result').iterator(): + protected.update(m[2] for m in ASSET.finditer(json.dumps([task.input_data, task.result]))) + for path in root().glob('*.json'): + meta = _metadata(path.stem) + if meta['touched_at'] >= cutoff or path.stem in protected: + continue + expired.append(path.stem) + if apply: + (root() / (path.stem + '.bin')).unlink(missing_ok=True) + path.unlink() + return len(expired) diff --git a/apps/system_manage/ai_hub_knowledge.py b/apps/system_manage/ai_hub_knowledge.py new file mode 100644 index 000000000..10458078e --- /dev/null +++ b/apps/system_manage/ai_hub_knowledge.py @@ -0,0 +1,54 @@ +"""Knowledge import uses the same authorized parser and durable task result as mindmaps.""" +from django.db import transaction +from django.utils import timezone +from rest_framework.exceptions import PermissionDenied, ValidationError +from knowledge.models import Knowledge +from system_manage import ai_hub as hub + + +@transaction.atomic +def parser_binding(workspace, knowledge_id, user, data=None): + knowledge = Knowledge.objects.select_for_update().filter(id=knowledge_id, workspace_id=workspace).first() + if not knowledge: + raise ValidationError('知识库不存在') + meta = dict(knowledge.meta or {}) + if data is not None: + parser_id = data.get('parser_id') + if parser_id: + parser = hub.one('parsers', parser_id, workspace, user) + if not parser.enabled: + raise ValidationError('解析服务已停用') + meta['parser_id'] = str(parser_id) if parser_id else '' + knowledge.meta = meta + knowledge.save(update_fields=['meta', 'update_time']) + return {'parser_id': meta.get('parser_id', '')} + + +def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter, limit, overlap): + knowledge = Knowledge.objects.get(id=knowledge_id, workspace_id=workspace) + parser_id = (knowledge.meta or {}).get('parser_id') + if not parser_id: + return None + if user is None: + raise PermissionDenied('解析服务调用需要用户身份') + from system_manage.ai_hub_worker import execute + # Mark as running before commit: the asynchronous consumer cannot claim the same task. + with transaction.atomic(): + task = hub.submit(workspace, user, {'name': file.name[:128], 'nodes': [ + {'type': 'parse', 'resource_id': parser_id}, {'type': 'render'}]}, upload=file) + task.status = 'RUNNING' + task.started_at = timezone.now() + task.save(update_fields=['status', 'started_at']) + execute(task) + file.seek(0) + if task.status != 'SUCCESS': + raise ValidationError(task.error or '解析失败') + from django.core.files.uploadedfile import SimpleUploadedFile + from common.handle.impl.text.text_split_handle import TextSplitHandle + content = task.result['markdown'].encode('utf-8') + converted = SimpleUploadedFile(file.name + '.md', content) + result = TextSplitHandle().handle(converted, patterns, with_filter, limit, + lambda _: content, lambda _: None, overlap) + result['name'] = file.name + return result, {'resource_id': str(parser_id), 'task_id': str(task.id), + 'resource_version': task.snapshot['nodes'][0]['resource_version']} diff --git a/apps/system_manage/ai_hub_parser_v1.py b/apps/system_manage/ai_hub_parser_v1.py new file mode 100644 index 000000000..4ee7128fc --- /dev/null +++ b/apps/system_manage/ai_hub_parser_v1.py @@ -0,0 +1,59 @@ +"""MinerU v1 task API adapter. Only follows paths built from the registered endpoint.""" +import re +import time +import requests +from rest_framework.exceptions import ValidationError + + +def _json(response): + if response.status_code not in (200, 201, 202): + raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})') + try: + body = response.json() + except ValueError as exc: + raise ValidationError('解析服务返回格式无效') from exc + if not isinstance(body, dict): + raise ValidationError('解析服务返回格式无效') + return body + + +def parse(parser, name, content, options, headers, progress_callback=None): + from system_manage.ai_hub import text, MAX_INLINE_TEXT + endpoint = parser.endpoint.rstrip('/') + deadline = time.monotonic() + 300 + try: + response = requests.post(endpoint + '/tasks', files={'files': (name, content)}, headers=headers, + params=[('tier', options.get('tier', 'standard')), ('ocr_mode', 'ocr' if options.get('ocr') else 'auto'), + ('output_formats', 'markdown'), ('output_formats', 'middle_json')], + timeout=(10, 30), allow_redirects=False) + task = _json(response) + from system_manage.ai_hub_progress import parser_progress + task_id = task.get('task_id', '') + if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id): + raise ValidationError('解析服务未返回有效任务ID') + if progress_callback: + progress_callback(parser_progress(task)) + while task.get('status') in ('queued', 'running'): + if time.monotonic() >= deadline: + raise ValidationError('解析等待超时,后台任务ID:' + task_id) + time.sleep(2) + task = _json(requests.get(endpoint + '/tasks/' + task_id, headers=headers, + timeout=(10, 20), allow_redirects=False)) + if progress_callback: + progress_callback(parser_progress(task)) + if task.get('status') != 'completed': + raise ValidationError('解析任务未成功,后台任务ID:' + task_id) + result = _json(requests.get(endpoint + '/tasks/' + task_id + '/result', headers=headers, + timeout=(10, 30), allow_redirects=False)) + except requests.RequestException as exc: + raise ValidationError('解析服务连接失败或超时') from exc + files = result.get('files', []) + if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) or files[0].get('status') != 'completed': + raise ValidationError('解析文件结果无效') + output = files[0].get('content', {}) + if not isinstance(output, dict): + raise ValidationError('解析文件内容无效') + markdown = text(output.get('markdown'), '解析结果', limit=MAX_INLINE_TEXT) + return {'markdown': markdown, 'parser_id': str(parser.id), 'images': output.get('images', {}), + 'source': {'task_id': task_id, 'parse': files[0].get('parse'), + 'middle_json': output.get('middle_json', {})}} diff --git a/apps/system_manage/ai_hub_progress.py b/apps/system_manage/ai_hub_progress.py new file mode 100644 index 000000000..dd030224e --- /dev/null +++ b/apps/system_manage/ai_hub_progress.py @@ -0,0 +1,37 @@ +"""Expose real phases/counts only. A single-file v1 job has no page percentage.""" +from django.utils import timezone + +LABELS = {'parse': '文档解析中', 'organize': '智能整理中', 'render': '生成思维导图'} + + +def parser_progress(task): + state = task.get('status') + phase = {'queued': '等待解析服务处理', 'running': '正在解析文档', 'completed': '获取解析结果'}.get(state, '正在解析文档') + raw = task.get('progress') or {} + def count(key): + value = raw.get(key, 0) if isinstance(raw, dict) else 0 + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0 + completed, total = count('completed'), count('total') + percent = min(100, completed * 100 // total) if total > 1 and completed else None + if total > 1: + phase += f'({min(completed, total)}/{total} 个文件)' + return {'stage': phase, 'percent': percent, 'completed_files': completed, 'total_files': total} + + +def task_progress(task): + elapsed = max(0, int(((task.finished_at or timezone.now()) - (task.started_at or task.created_at)).total_seconds())) + if task.status == 'SUCCESS': + return {'stage': '完成', 'percent': 100, 'elapsed_seconds': elapsed} + if task.status == 'PENDING': + return {'stage': '等待任务处理', 'percent': None, 'elapsed_seconds': elapsed} + if task.status == 'FAILED': + return {'stage': '处理失败', 'percent': None, 'elapsed_seconds': elapsed} + current = next((step for step in task.steps if step.get('status') == 'RUNNING'), {}) + detail = current.get('progress') or {} + percent = detail.get('percent') + total_steps = max(1, len(task.snapshot.get('nodes', []))) + complete_steps = sum(step.get('status') == 'SUCCESS' for step in task.steps) + if percent is not None: + percent = min(99, int((complete_steps * 100 + percent) / total_steps)) + return {'stage': detail.get('stage') or LABELS.get(current.get('type'), '处理中'), + 'percent': percent, 'elapsed_seconds': elapsed} diff --git a/apps/system_manage/ai_hub_worker.py b/apps/system_manage/ai_hub_worker.py new file mode 100644 index 000000000..eaff5de81 --- /dev/null +++ b/apps/system_manage/ai_hub_worker.py @@ -0,0 +1,104 @@ +"""Database-backed single consumer; never consumes legacy Celery queues.""" +from django.db import transaction +from django.utils import timezone +from system_manage.models.ai_hub import AITask +from system_manage import ai_hub as hub + + +def claim(): + with transaction.atomic(): + task = AITask.objects.select_for_update(skip_locked=True).filter(status='PENDING').order_by('created_at').first() + if not task: + return None + task.status = 'RUNNING' + task.started_at = timezone.now() + task.save(update_fields=['status', 'started_at', 'updated_at']) + return task + + +def execute(task): + from users.models import User + steps = [] + try: + user = User.objects.get(pk=task.owner_id) + if not user.is_active: + raise ValueError('用户已停用') + if task.service_id: + service = hub.one('services', task.service_id, task.workspace_id, user) + if not service.enabled: + raise ValueError('服务已停用') + markdown = task.input_data.get('markdown', '') + original = markdown + source = {} + tree = {} + task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree} + task.save(update_fields=['result', 'updated_at']) + for node in task.snapshot['nodes']: + step = {'type': node['type'], 'status': 'RUNNING'} + steps.append(step) + task.steps = steps + task.save(update_fields=['steps', 'updated_at']) + if node['type'] == 'parse': + parser = hub.one('parsers', node['resource_id'], task.workspace_id, user) + if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']: + raise ValueError('解析服务已停用或版本发生变化,请重新提交') + def report_progress(progress): + step['progress'] = progress + AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update( + steps=steps, updated_at=timezone.now()) + report_progress({'stage': '上传文档到解析服务', 'percent': None}) + parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress) + from system_manage.ai_hub_assets import asset_lock, normalize + with asset_lock(): + markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果') + source = parsed['source'] + # Source JSON can contain image data too; preserve structure without inline payloads. + import json + source = json.loads(normalize(json.dumps(source), task.workspace_id, user)) + task.result = {'original': original, 'markdown': markdown, 'source': source} + task.save(update_fields=['result', 'updated_at']) + elif node['type'] == 'organize': + from models_provider.tools import get_model, get_model_default_params + from langchain_core.messages import HumanMessage, SystemMessage + model = hub.model_for(task.workspace_id, user, node['model_id']) + if model.update_time.isoformat() != node['resource_version']: + raise ValueError('模型配置已变更,请重新提交') + # Bound input; never silently truncate the original document. + from system_manage.ai_hub_assets import protect_images, restore_images + model_markdown, image_tokens = protect_images(markdown) + if len(model_markdown) > 60000: + raise ValueError('文档超过智能整理长度限制,请分批处理') + try: + instance = get_model(model, **{**get_model_default_params(model), + 'temperature': node.get('temperature', 0.2), 'timeout': 180}) + prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。' + response = instance.invoke([SystemMessage(content=prompt), HumanMessage(content=model_markdown)]) + except Exception as exc: + from rest_framework.exceptions import ValidationError + raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc + markdown = restore_images(hub.text(response.content, '整理结果'), image_tokens) + else: + tree = hub.markdown_tree(markdown) + step['status'] = 'SUCCESS' + task.steps = steps + # Persist intermediate original output so a later model failure does not lose OCR work. + task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree} + task.save(update_fields=['steps', 'result', 'updated_at']) + task.status = 'SUCCESS' + except Exception as exc: + if steps: + steps[-1]['status'] = 'FAILED' + # Do not echo URLs, credentials, request bodies or provider exceptions to users/logs. + from rest_framework.exceptions import APIException + if isinstance(exc, APIException): + detail = exc.detail + while isinstance(detail, (list, tuple, dict)) and detail: + detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0] + task.error = str(detail)[:512] + else: + task.error = str(exc)[:512] if isinstance(exc, ValueError) else '执行失败,请检查服务状态后重试' + task.status = 'FAILED' + task.steps = steps + task.finished_at = timezone.now() + task.save(update_fields=['status', 'error', 'steps', 'finished_at', 'updated_at']) + return task diff --git a/apps/system_manage/management/__init__.py b/apps/system_manage/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/system_manage/management/commands/__init__.py b/apps/system_manage/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/system_manage/management/commands/ai_hub_cleanup_assets.py b/apps/system_manage/management/commands/ai_hub_cleanup_assets.py new file mode 100644 index 000000000..99da5f959 --- /dev/null +++ b/apps/system_manage/management/commands/ai_hub_cleanup_assets.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand +from system_manage.ai_hub_assets import cleanup + + +class Command(BaseCommand): + help = 'Check expired Hub images; --apply removes only images not referenced by saved graphs/history.' + + def add_arguments(self, parser): + parser.add_argument('--apply', action='store_true') + + def handle(self, *args, **options): + count = cleanup(apply=options['apply']) + self.stdout.write(f'{"Removed" if options["apply"] else "Eligible"}: {count}') diff --git a/apps/system_manage/management/commands/ai_hub_worker.py b/apps/system_manage/management/commands/ai_hub_worker.py new file mode 100644 index 000000000..9af28f978 --- /dev/null +++ b/apps/system_manage/management/commands/ai_hub_worker.py @@ -0,0 +1,37 @@ +import time +from datetime import timedelta +from django.core.management.base import BaseCommand +from django.db import close_old_connections +from django.utils import timezone +from system_manage.models.ai_hub import AITask +from system_manage.ai_hub_worker import claim, execute + + +class Command(BaseCommand): + help = 'Run the isolated AI hub task consumer (does not run migrations).' + + def add_arguments(self, parser): + parser.add_argument('--once', action='store_true') + parser.add_argument('--recover', action='store_true', help='Mark running tasks older than 30 minutes failed') + + def handle(self, *args, **options): + if options['recover']: + AITask.objects.filter(status='RUNNING', updated_at__lt=timezone.now() - timedelta(minutes=30)).update( + status='FAILED', error='执行中断,请重试', finished_at=timezone.now()) + next_cleanup = time.monotonic() + 86400 + while True: + close_old_connections() + if time.monotonic() >= next_cleanup: + from system_manage.ai_hub_assets import cleanup + try: + cleanup(apply=True) + except Exception: + self.stderr.write('图片缓存清理失败,将于下次重试') + next_cleanup = time.monotonic() + 86400 + task = claim() + if task: + execute(task) + if options['once']: + return + if not task: + time.sleep(2) diff --git a/apps/system_manage/migrations/0007_ai_hub.py b/apps/system_manage/migrations/0007_ai_hub.py new file mode 100644 index 000000000..a3dea3e38 --- /dev/null +++ b/apps/system_manage/migrations/0007_ai_hub.py @@ -0,0 +1,115 @@ +# Generated by Django 5.2.8 on 2026-10-19 10:15 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('system_manage', '0006_accesscontrolpolicy_and_record'), + ] + + operations = [ + migrations.CreateModel( + name='AIFlow', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('owner_id', models.UUIDField(db_index=True)), + ('name', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('revision', models.PositiveIntegerField(default=1)), + ('nodes', models.JSONField(default=list)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AIParserResource', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('owner_id', models.UUIDField(db_index=True)), + ('name', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('endpoint', models.URLField(max_length=1024)), + ('credential', models.TextField(blank=True, default='')), + ('enabled', models.BooleanField(default=True)), + ('allowed_users', models.JSONField(default=list)), + ('options', models.JSONField(default=dict)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AIService', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('owner_id', models.UUIDField(db_index=True)), + ('name', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('enabled', models.BooleanField(default=True)), + ('allowed_users', models.JSONField(default=list)), + ('key_hash', models.CharField(blank=True, default='', max_length=64)), + ('requests_per_minute', models.PositiveIntegerField(default=20)), + ('flow', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='system_manage.aiflow')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AITask', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('owner_id', models.UUIDField(db_index=True)), + ('name', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('status', models.CharField(db_index=True, default='PENDING', max_length=16)), + ('snapshot', models.JSONField(default=dict)), + ('input_data', models.JSONField(default=dict)), + ('file_name', models.CharField(default='', max_length=255)), + ('file_data', models.BinaryField(null=True)), + ('result', models.JSONField(default=dict)), + ('steps', models.JSONField(default=list)), + ('error', models.CharField(default='', max_length=512)), + ('attempt', models.PositiveIntegerField(default=1)), + ('started_at', models.DateTimeField(null=True)), + ('finished_at', models.DateTimeField(null=True)), + ('service', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aiservice')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AIArtifact', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128)), + ('owner_id', models.UUIDField(db_index=True)), + ('name', models.CharField(max_length=128)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('revision', models.PositiveIntegerField(default=1)), + ('original', models.TextField(default='')), + ('markdown', models.TextField(default='')), + ('layout', models.CharField(default='tree', max_length=32)), + ('history', models.JSONField(default=list)), + ('task', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aitask')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/apps/system_manage/models/ai_hub.py b/apps/system_manage/models/ai_hub.py new file mode 100644 index 000000000..b7e58185e --- /dev/null +++ b/apps/system_manage/models/ai_hub.py @@ -0,0 +1,60 @@ +"""Shared AI resources, versioned flows and durable user results.""" +import uuid +from django.db import models + + +class HubBase(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + workspace_id = models.CharField(max_length=128, default='default', db_index=True) + owner_id = models.UUIDField(db_index=True) + name = models.CharField(max_length=128) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + abstract = True + + +class AIParserResource(HubBase): + endpoint = models.URLField(max_length=1024) + credential = models.TextField(default='', blank=True) + enabled = models.BooleanField(default=True) + allowed_users = models.JSONField(default=list) + options = models.JSONField(default=dict) + + +class AIFlow(HubBase): + revision = models.PositiveIntegerField(default=1) + nodes = models.JSONField(default=list) + + +class AIService(HubBase): + flow = models.ForeignKey(AIFlow, on_delete=models.PROTECT) + enabled = models.BooleanField(default=True) + allowed_users = models.JSONField(default=list) + key_hash = models.CharField(max_length=64, default='', blank=True) + requests_per_minute = models.PositiveIntegerField(default=20) + + +class AITask(HubBase): + service = models.ForeignKey(AIService, null=True, on_delete=models.SET_NULL) + status = models.CharField(max_length=16, default='PENDING', db_index=True) + snapshot = models.JSONField(default=dict) + input_data = models.JSONField(default=dict) + file_name = models.CharField(max_length=255, default='') + file_data = models.BinaryField(null=True) + result = models.JSONField(default=dict) + steps = models.JSONField(default=list) + error = models.CharField(max_length=512, default='') + attempt = models.PositiveIntegerField(default=1) + started_at = models.DateTimeField(null=True) + finished_at = models.DateTimeField(null=True) + + +class AIArtifact(HubBase): + task = models.ForeignKey(AITask, null=True, on_delete=models.SET_NULL) + revision = models.PositiveIntegerField(default=1) + original = models.TextField(default='') + markdown = models.TextField(default='') + layout = models.CharField(max_length=32, default='tree') + history = models.JSONField(default=list) diff --git a/apps/system_manage/test_ai_hub.py b/apps/system_manage/test_ai_hub.py new file mode 100644 index 000000000..494ac8ed6 --- /dev/null +++ b/apps/system_manage/test_ai_hub.py @@ -0,0 +1,462 @@ +import json +from unittest import TestCase +from unittest.mock import patch, Mock +from django.db import transaction +from django.core.files.uploadedfile import SimpleUploadedFile +from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound +from users.models import User +from users.models.user_manage_ext import UserActivationApplication +from system_manage import ai_hub as hub +from system_manage.ai_hub_worker import claim, execute +from system_manage.models.ai_hub import AIFlow, AIParserResource, AITask, AIArtifact, AIService + + +class HubTests(TestCase): + def setUp(self): + import tempfile + from django.test import override_settings + from pathlib import Path + self.asset_temp = tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[3] / '.codex_tmp') + self.addCleanup(self.asset_temp.cleanup) + self.asset_settings = override_settings(AI_HUB_ASSET_DIR=self.asset_temp.name) + self.asset_settings.enable() + self.addCleanup(self.asset_settings.disable) + self.atomic = transaction.atomic() + self.atomic.__enter__() + self.admin = User.objects.create(username='admin', nick_name='admin', role='ADMIN', is_active=True) + self.user = User.objects.create(username='user', nick_name='user', role='USER', is_active=True) + self.other = User.objects.create(username='other', nick_name='other', role='USER', is_active=True) + + def tearDown(self): + transaction.set_rollback(True) + self.atomic.__exit__(None, None, None) + + def parser(self, **kwargs): + return AIParserResource.objects.create(workspace_id='default', owner_id=self.admin.id, + name='测试解析', endpoint='http://parser.test/api', allowed_users=[str(self.user.id)], **kwargs) + + def test_parser_grant_and_workspace_isolation(self): + p = self.parser() + self.assertEqual(hub.one('parsers', p.id, 'default', self.user).id, p.id) + with self.assertRaises(NotFound): hub.one('parsers', p.id, 'default', self.other) + with self.assertRaises(NotFound): hub.one('parsers', p.id, 'other', self.admin) + with self.assertRaises(PermissionDenied): hub.one('parsers', p.id, 'default', self.user, edit=True) + + def test_only_admin_can_register_parser(self): + with self.assertRaises(PermissionDenied): hub.parser_save('default', self.user, {}) + + def test_authorization_excludes_rejected_pending_disabled(self): + application = UserActivationApplication.objects.create(user=self.user, status='REJECTED') + with self.assertRaises(ValidationError): hub.users([str(self.user.id)]) + application.status = 'PENDING'; application.save() + with self.assertRaises(ValidationError): hub.users([str(self.user.id)]) + application.status = 'APPROVED'; application.save() + self.assertEqual(hub.users([str(self.user.id)]), [str(self.user.id)]) + self.user.is_active = False; self.user.save() + with self.assertRaises(ValidationError): hub.users([str(self.user.id)]) + + def test_parser_secret_never_serialized_and_preserved(self): + with patch('common.utils.rsa_util.rsa_long_encrypt', return_value='encrypted'): + p = hub.parser_save('default', self.admin, {'name': '解析', 'endpoint': 'http://parser.test/api', 'api_key': 'private', 'allowed_users': []}) + self.assertEqual(p.credential, 'encrypted') + self.assertNotIn('private', json.dumps(hub.serialize(p))) + self.assertNotIn('encrypted', json.dumps(hub.serialize(p))) + p = hub.parser_save('default', self.admin, {'name': '解析2'}, p) + self.assertEqual(p.credential, 'encrypted') + + def test_parser_url_and_options_validation(self): + for endpoint in ('file:///tmp/a', 'http://user:secret@server/api', 'http://server/api?key=secret'): + with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': endpoint}) + with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/api', 'options': {'server_url': 'http://other'}}) + + def test_parser_toggle_preserves_shared_options_and_grants(self): + p = hub.parser_save('default', self.admin, {'name': 'shared', 'endpoint': 'http://server/v1', + 'allowed_users': [str(self.user.id)], 'options': {'protocol': 'v1', 'tier': 'basic'}}) + self.user.is_active = False + self.user.save(update_fields=['is_active']) + hub.parser_save('default', self.admin, {'enabled': False}, p) + p.refresh_from_db() + self.assertFalse(p.enabled) + self.assertEqual(p.allowed_users, [str(self.user.id)]) + self.assertEqual(p.options, {'protocol': 'v1', 'tier': 'basic'}) + self.assertEqual(p.name, 'shared') + + @patch('requests.get') + def test_parser_connection_checks_protocol_and_does_not_return_service_content(self, get): + p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}}) + get.return_value = Mock(status_code=200) + get.return_value.json.return_value = {'paths': {'/v1/tasks': {'post': {}}}, 'secret': 'not-for-browser'} + result = hub.parser_test(p) + self.assertTrue(result['connected']) + self.assertNotIn('secret', result) + self.assertEqual(get.call_args.args[0], 'http://server/openapi.json') + self.assertFalse(get.call_args.kwargs['allow_redirects']) + get.return_value.json.return_value = {'paths': {'/v1/tasks': {'get': {}}}} + with self.assertRaises(ValidationError): hub.parser_test(p) + p.endpoint = 'http://server/api'; p.options = {'protocol': 'legacy'} + get.return_value.json.return_value = {'paths': {'/file_parse': {'post': {}}}} + self.assertTrue(hub.parser_test(p)['connected']) + + @patch('requests.get') + def test_parser_connection_failures_and_permissions(self, get): + import requests + from types import SimpleNamespace + from system_manage.views.ai_hub import ParserTest + p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1'}) + with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id) + get.assert_not_called() + get.return_value = Mock(status_code=302) + with self.assertRaises(ValidationError): hub.parser_test(p) + get.return_value = Mock(status_code=200) + get.return_value.json.side_effect = ValueError('sensitive server response') + with self.assertRaises(ValidationError) as error: hub.parser_test(p) + self.assertNotIn('sensitive', str(error.exception)) + get.side_effect = requests.Timeout('sensitive URL') + with self.assertRaises(ValidationError) as error: hub.parser_test(p) + self.assertNotIn('sensitive', str(error.exception)) + + def test_flow_versions_are_immutable(self): + a = hub.flow_create('default', self.user, {'name': '文档', 'nodes': [{'type': 'render'}]}) + b = hub.flow_create('default', self.user, {'name': '文档', 'previous_id': str(a.id), 'nodes': [{'type': 'render'}]}) + self.assertEqual((a.revision, b.revision), (1, 2)) + self.assertNotEqual(a.id, b.id) + + def test_invalid_flow_and_ungranted_parser_rejected(self): + p = self.parser() + for nodes in ([{'type': 'render'}, {'type': 'parse', 'resource_id': str(p.id)}], [{'type': 'custom'}], [{'type': 'render'}, {'type': 'render'}]): + with self.assertRaises(ValidationError): hub.validate_nodes('default', self.user, nodes) + with self.assertRaises(NotFound): hub.validate_nodes('default', self.other, [{'type': 'parse', 'resource_id': str(p.id)}, {'type': 'render'}]) + + def test_submit_claim_and_rule_render(self): + task = hub.submit('default', self.user, {'markdown': '# 标题\n## 内容\n事实42\n```\n# 不作标题\n```', 'nodes': [{'type': 'render'}]}) + picked = claim() + self.assertEqual(picked.id, task.id) + self.assertIsNone(claim()) + execute(picked); picked.refresh_from_db() + self.assertEqual(picked.status, 'SUCCESS') + tree = picked.result['tree'] + self.assertEqual(tree['children'][0]['text'], '标题') + self.assertNotIn('不作标题', json.dumps(tree, ensure_ascii=False)) + + def test_parser_task_preserves_original_and_source(self): + parser = self.parser() + task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('scan.pdf', b'pdf')) + response = Mock(status_code=200) + response.json.return_value = {'results': {'scan': {'md_content': '# 内容\n数值42', 'middle_json': {'pages': [1]}}}} + with patch('requests.post', return_value=response) as post: + execute(claim()) + self.assertFalse(post.call_args.kwargs['allow_redirects']) + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS') + self.assertEqual(task.result['original'], '# 内容\n数值42') + self.assertEqual(task.result['source']['pages'], [1]) + + def test_parser_disabled_or_changed_after_submit_fails(self): + parser = self.parser() + task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf')) + parser.enabled = False; parser.save() + with patch('requests.post') as post: execute(claim()); post.assert_not_called() + task.refresh_from_db() + self.assertEqual(task.status, 'FAILED') + + def test_model_calls_existing_provider_with_selected_parameters(self): + from models_provider.models import Model + model = Model.objects.create(name='整理', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[]) + task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id), 'temperature': 0}, {'type': 'render'}]}) + fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿') + with patch('models_provider.tools.get_model', return_value=fake) as get: + execute(claim()) + self.assertEqual(get.call_args.kwargs['temperature'], 0) + task.refresh_from_db() + self.assertEqual(task.result['original'], '# 原稿') + self.assertEqual(task.result['markdown'], '# 整理稿') + + def test_provider_exception_is_sanitized(self): + parser = self.parser() + task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf')) + with patch('requests.post', side_effect=RuntimeError('private-url-key')): execute(claim()) + task.refresh_from_db(); self.assertEqual(task.status, 'FAILED') + self.assertNotIn('private', task.error) + + def test_empty_parse_does_not_succeed(self): + parser = self.parser() + response = Mock(status_code=200); response.json.return_value = {'results': {'doc': {'md_content': ''}}} + with patch('requests.post', return_value=response): + with self.assertRaises(ValidationError): hub.parse_file(parser, 'x.pdf', b'pdf') + + def test_task_validation_error_is_displayed_without_framework_details(self): + parser = self.parser() + task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf')) + with patch('system_manage.ai_hub.parse_file', side_effect=ValidationError('解析失败,请检查文档')): + execute(claim()) + task.refresh_from_db() + self.assertEqual(task.error, '解析失败,请检查文档') + + def test_artifact_review_preserves_original_and_detects_conflicts(self): + a = hub.artifact_save('default', self.user, {'name': '图谱', 'original': '# OCR原值', 'markdown': '# 复核结果'}) + a = hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'original': '篡改原稿', 'markdown': '# 第二版'}, a) + self.assertEqual(a.original, '# OCR原值'); self.assertEqual(a.revision, 2) + self.assertEqual(a.history[0]['markdown'], '# 复核结果') + with self.assertRaises(ValidationError): hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'markdown': '# 旧提交'}, a) + with self.assertRaises(NotFound): hub.one('artifacts', a.id, 'default', self.other) + + def test_published_service_is_version_pinned_and_disabled_stops_submit(self): + flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]}) + service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]}) + task = hub.submit('default', self.user, {'markdown': '# 文档'}, service=service) + self.assertEqual(task.snapshot['flow_revision'], 1) + service.enabled = False; service.save() + with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service) + execute(claim()); task.refresh_from_db(); self.assertEqual(task.status, 'FAILED') + + def test_unknown_ids_and_missing_inputs_are_validation_errors(self): + with self.assertRaises(ValidationError): hub.one('parsers', None, 'default', self.admin) + with self.assertRaises(ValidationError): hub.submit('default', self.user, {'nodes': [{'type': 'render'}]}) + with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': 'x', 'nodes': [{'type': 'parse', 'resource_id': str(self.parser().id)}, {'type': 'render'}]}) + + def api(self, method, kind, data=None, pk=None, user=None, headers=None): + from rest_framework.test import APIRequestFactory, force_authenticate + from system_manage.views.ai_hub import HubView, PublishedRun + request = getattr(APIRequestFactory(), method)('/', data or {}, format='json', **(headers or {})) + force_authenticate(request, user=user or self.admin) + kwargs = {'workspace_id': 'default'} + if kind != 'published': kwargs['kind'] = kind + if pk: kwargs['pk'] = pk + view = PublishedRun if kind == 'published' else HubView + with patch('users.serializers.user.get_workspace_list_by_user', return_value=[{'id': 'default'}]): + response = view.as_view()(request, **kwargs) + return json.loads(response.content) if hasattr(response, 'content') else response.data + + def test_api_service_key_rotation_revocation_and_real_task(self): + flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]}) + service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]}) + rotated = self.api('post', 'services', {'rotate_key': True}, service.id)['data'] + self.assertTrue(rotated['api_key']) + self.assertNotIn('api_key', self.api('get', 'services', pk=service.id)['data']) + denied = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user) + self.assertNotEqual(denied['code'], 200) + accepted = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user, {'HTTP_X_SERVICE_KEY': rotated['api_key']}) + self.assertEqual(accepted['code'], 200) + self.assertEqual(AITask.objects.get(id=accepted['data']['id']).status, 'PENDING') + self.api('post', 'services', {'revoke_key': True}, service.id) + service.refresh_from_db(); self.assertEqual(service.key_hash, '') + + def test_api_cannot_edit_shared_service_and_stops_disabled_service(self): + flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]}) + service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]}) + response = self.api('post', 'services', {'enabled': False}, service.id, self.user) + self.assertNotEqual(response['code'], 200) + self.api('post', 'services', {'enabled': False}, service.id) + response = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user) + self.assertNotEqual(response['code'], 200) + + def test_api_retry_resets_failure_without_duplicate_task(self): + task = hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]}) + task.status = 'FAILED'; task.error = '中断'; task.save() + response = self.api('post', 'tasks', {}, task.id, self.user) + self.assertEqual(response['code'], 200) + task.refresh_from_db(); self.assertEqual(task.attempt, 2); self.assertEqual(task.status, 'PENDING') + self.assertEqual(AITask.objects.count(), 1) + response = self.api('post', 'tasks', {}, task.id, self.user) + self.assertNotEqual(response['code'], 200) + + def test_knowledge_binding_and_shared_parser_execution(self): + from knowledge.models import KnowledgeFolder, Knowledge + from system_manage.ai_hub_knowledge import parser_binding, split_with_parser + KnowledgeFolder.objects.create(id='default', name='root') + knowledge = Knowledge.objects.create(name='测试知识库', desc='', folder_id='default', meta={'segmentation': {'chunk_size': 100}}) + parser = self.parser() + parser_binding('default', knowledge.id, self.user, {'parser_id': str(parser.id)}) + knowledge.refresh_from_db(); self.assertEqual(knowledge.meta['segmentation']['chunk_size'], 100) + with self.assertRaises(NotFound): parser_binding('default', knowledge.id, self.other, {'parser_id': str(parser.id)}) + response = Mock(status_code=200) + response.json.return_value = {'results': {'scan': {'md_content': '# 章节\n识别结果42', 'middle_json': {'pages': [1]}}}} + file = SimpleUploadedFile('scan.pdf', b'pdf') + with patch('requests.post', return_value=response): + result, provenance = split_with_parser('default', knowledge.id, self.user, file, None, False, 100, 20) + task = AITask.objects.get(id=provenance['task_id']) + self.assertEqual(task.status, 'SUCCESS'); self.assertEqual(provenance['resource_id'], str(parser.id)) + self.assertIn('识别结果42', json.dumps(result, ensure_ascii=False)); self.assertEqual(file.tell(), 0) + parser_binding('default', knowledge.id, self.user, {'parser_id': ''}) + self.assertIsNone(split_with_parser('default', knowledge.id, None, file, None, False, 100, 0)) + + def test_model_selection_follows_view_grants_instead_of_ownership(self): + from models_provider.models import Model + from types import SimpleNamespace + model = Model.objects.create(name='私有', model_type='LLM', model_name='demo', provider='test', user_id=self.user.id, credential='', status='SUCCESS', model_params_form=[]) + with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[]): + with self.assertRaises(PermissionDenied): hub.model_for('default', self.user, model.id) + grant = SimpleNamespace(target=str(model.id), permission_list=['VIEW']) + with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[grant]): + self.assertEqual(hub.model_for('default', self.user, model.id).id, model.id) + + def test_queue_limit_applies_to_published_service_too(self): + flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]}) + service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]}) + for _ in range(10): + hub.submit('default', self.user, {'markdown': '# 文档'}, service=service) + with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service) + self.assertEqual(AITask.objects.count(), 10) + + def test_stopped_user_cannot_submit_directly(self): + self.user.is_active = False; self.user.save() + with self.assertRaises(PermissionDenied): hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]}) + + def test_v1_adapter_polls_registered_endpoint_and_preserves_source(self): + parser = self.parser(options={'protocol': 'v1', 'tier': 'flash'}) + created = Mock(status_code=202); created.json.return_value = {'task_id': 'task_test', 'status': 'queued', 'result_url': 'http://untrusted/secret'} + status = Mock(status_code=200); status.json.return_value = {'task_id': 'task_test', 'status': 'completed'} + result = Mock(status_code=200); result.json.return_value = {'files': [{'status': 'completed', 'parse': {'engine': 'native'}, 'content': {'markdown': '# 识别结果42', 'middle_json': {'pages': [1]}}}]} + with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=created) as post, patch('system_manage.ai_hub_parser_v1.requests.get', side_effect=[status, result]) as get, patch('system_manage.ai_hub_parser_v1.time.sleep'): + parsed = hub.parse_file(parser, 'test.pdf', b'pdf') + self.assertEqual(parsed['markdown'], '# 识别结果42') + self.assertEqual(parsed['source']['middle_json']['pages'], [1]) + self.assertEqual(get.call_args.args[0], parser.endpoint + '/tasks/task_test/result') + self.assertIn(('tier', 'flash'), post.call_args.kwargs['params']) + + def test_v1_adapter_rejects_external_task_paths_and_failed_files(self): + parser = self.parser(options={'protocol': 'v1'}) + response = Mock(status_code=202); response.json.return_value = {'task_id': '../../other', 'status': 'completed'} + with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=response), patch('system_manage.ai_hub_parser_v1.requests.get') as get: + with self.assertRaises(ValidationError): hub.parse_file(parser, 'test.pdf', b'pdf') + get.assert_not_called() + + def image_markdown(self): + import base64 + import io + from PIL import Image + content = io.BytesIO() + Image.new('RGB', (40, 20), '#2468ac').save(content, 'PNG') + return '# 图片验收\n![结构图](data:image/png;base64,' + base64.b64encode(content.getvalue()).decode() + ')', content.getvalue() + + def test_asset_submission_persistence_deduplication_and_isolation(self): + from system_manage.ai_hub_assets import ASSET, image_bytes, root + markdown, content = self.image_markdown() + task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]}) + url = ASSET.search(task.input_data['markdown']) + self.assertIsNotNone(url) + self.assertNotIn('base64', task.input_data['markdown']) + self.assertEqual(image_bytes('default', self.user, url[2])[0], content) + with self.assertRaises(NotFound): image_bytes('default', self.other, url[2]) + with self.assertRaises(NotFound): image_bytes('other', self.admin, url[2]) + with self.assertRaises(NotFound): hub.submit('default', self.other, {'markdown': task.input_data['markdown'], 'nodes': [{'type': 'render'}]}) + hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]}) + self.assertEqual(len(list(root().glob('*.bin'))), 1) + + def test_asset_cleanup_preserves_original_history_and_inflight(self): + import time + from system_manage.ai_hub_assets import ASSET, cleanup, root + markdown, _ = self.image_markdown() + artifact = hub.artifact_save('default', self.user, {'name': '测试图片', 'markdown': markdown, 'original': '', 'layout': 'tree'}) + asset_id = ASSET.search(artifact.markdown)[2] + hub.artifact_save('default', self.user, {'name': artifact.name, 'revision': artifact.revision, 'markdown': '# 删除当前引用'}, artifact) + self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0) + self.assertTrue((root() / (asset_id + '.bin')).exists()) + # Only remove the isolated test record; existing shared records are never involved. + artifact.delete() + task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]}) + self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0) + execute(task) + self.assertEqual(cleanup(now=time.time() + 8 * 86400), 1) + self.assertTrue((root() / (asset_id + '.bin')).exists()) + self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 1) + self.assertFalse((root() / (asset_id + '.bin')).exists()) + + def test_asset_parser_relative_images_and_invalid_payload(self): + from system_manage.ai_hub_assets import asset_lock, normalize, ASSET + markdown, _ = self.image_markdown() + uri = markdown.split('](', 1)[1][:-1] + with asset_lock(): + normalized = normalize('![图](images/a.png)\n', 'default', self.user, {'a.png': uri}) + self.assertEqual(len(ASSET.findall(normalized)), 2) + with self.assertRaises(ValidationError): normalize('![坏图](data:image/png;base64,aGVsbG8=)', 'default', self.user) + + def test_asset_bundle_is_offline_and_model_cannot_drop_images(self): + import io + import zipfile + from types import SimpleNamespace + from system_manage.views.ai_hub import MarkdownExport + from system_manage.ai_hub_assets import protect_images, restore_images + markdown, content = self.image_markdown() + artifact = hub.artifact_save('default', self.user, {'name': '测试导出', 'markdown': markdown}) + response = MarkdownExport().post(SimpleNamespace(user=self.user, data={'markdown': artifact.markdown}), 'default') + with zipfile.ZipFile(io.BytesIO(response.content)) as archive: + self.assertEqual(len(archive.namelist()), 2) + exported = archive.read('document.md').decode() + self.assertNotIn('/admin/api/', exported) + self.assertIn('images/', exported) + self.assertEqual(archive.read(next(n for n in archive.namelist() if n.startswith('images/'))), content) + protected, images = protect_images(artifact.markdown) + self.assertNotIn('/admin/api/', protected) + self.assertEqual(restore_images(protected, images), artifact.markdown) + self.assertIn(images[0], restore_images('# 整理完成但遗漏图片', images)) + self.assertEqual(restore_images(protected + protected, images).count(images[0]), 1) + self.assertIn(images[0], restore_images('AIHUBIMAGE 0001 TOKEN', images)) + + def test_organize_restores_eight_omitted_images_and_preserves_original(self): + from models_provider.models import Model + markdown, _ = self.image_markdown() + image = markdown.split('\n', 1)[1] + original = '# 报告\n' + '\n\n'.join(image for _ in range(8)) + model = Model.objects.create(name='测试整理', model_type='LLM', model_name='demo', provider='test', + user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[]) + task = hub.submit('default', self.admin, {'markdown': original, 'nodes': [ + {'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]}) + normalized = task.input_data['markdown'] + fake = Mock(); fake.invoke.return_value = Mock(content='# 整理结果\n- 结论42') + with patch('models_provider.tools.get_model', return_value=fake): + execute(claim()) + task.refresh_from_db() + self.assertEqual(task.status, 'SUCCESS', task.error) + self.assertEqual(task.result['original'], normalized) + self.assertEqual(task.result['markdown'].count('![结构图]('), 8) + self.assertNotIn('AIHUBIMAGE', task.result['markdown']) + self.assertIn('结论42', task.result['markdown']) + self.assertNotIn('/admin/api/', fake.invoke.call_args.args[0][1].content) + + def test_failed_organize_keeps_source_available(self): + from models_provider.models import Model + model = Model.objects.create(name='测试失败', model_type='LLM', model_name='demo', provider='test', + user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[]) + task = hub.submit('default', self.admin, {'markdown': '# 保留原稿42', 'nodes': [ + {'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]}) + with patch('models_provider.tools.get_model', side_effect=RuntimeError('private')): execute(claim()) + task.refresh_from_db() + self.assertEqual(task.status, 'FAILED') + self.assertEqual(task.result['original'], '# 保留原稿42') + self.assertNotIn('private', task.error) + + def test_single_file_progress_is_indeterminate_and_counts_are_real(self): + from system_manage.ai_hub_progress import parser_progress, task_progress + single = parser_progress({'status': 'running', 'progress': {'completed': 0, 'total': 1}}) + self.assertIsNone(single['percent']) + self.assertEqual(single['stage'], '正在解析文档') + self.assertEqual(parser_progress({'status': 'queued'})['stage'], '等待解析服务处理') + self.assertEqual(parser_progress({'status': 'running', 'progress': {'completed': 2, 'total': 4}})['percent'], 50) + task = hub.submit('default', self.user, {'markdown': '# 测试', 'nodes': [{'type': 'render'}]}) + self.assertIsNone(task_progress(task)['percent']) + task.status = 'RUNNING'; task.steps = [{'type': 'parse', 'status': 'RUNNING', 'progress': single}] + self.assertEqual(hub.serialize(task, True)['progress']['stage'], '正在解析文档') + self.assertIsNone(hub.serialize(task, True)['progress']['percent']) + + def test_v1_progress_reaches_database_before_task_finishes(self): + parser = self.parser(options={'protocol': 'v1'}) + task = hub.submit('default', self.user, {'nodes': [ + {'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('test.pdf', b'pdf')) + created = Mock(status_code=202); created.json.return_value = {'task_id': 'test_progress', 'status': 'queued'} + running = Mock(status_code=200); running.json.return_value = {'status': 'running', 'progress': {'completed': 0, 'total': 1}} + done = Mock(status_code=200); done.json.return_value = {'status': 'completed'} + result = Mock(status_code=200); result.json.return_value = {'files': [ + {'status': 'completed', 'content': {'markdown': '# 完成42'}}]} + seen = [] + responses = iter([running, done, result]) + def get(*args, **kwargs): + row = AITask.objects.get(pk=task.pk) + seen.append(hub.serialize(row, True)['progress']) + return next(responses) + with patch('requests.post', return_value=created), patch('requests.get', side_effect=get), patch('system_manage.ai_hub_parser_v1.time.sleep'): + execute(claim()) + self.assertEqual([item['stage'] for item in seen], ['等待解析服务处理', '正在解析文档', '获取解析结果']) + self.assertTrue(all(item['percent'] is None for item in seen)) + task.refresh_from_db() + self.assertEqual(hub.serialize(task, True)['progress']['percent'], 100) diff --git a/apps/system_manage/urls.py b/apps/system_manage/urls.py index 4d60a11cc..f3f2cef21 100644 --- a/apps/system_manage/urls.py +++ b/apps/system_manage/urls.py @@ -1,11 +1,13 @@ from django.urls import path from . import views -from .views.ai_hub import HubView, PublishedRun, ParserTest +from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport app_name = "system_manage" # @formatter:off urlpatterns = [ + path('workspace//ai_hub/exports', MarkdownExport.as_view()), + path('workspace//ai_hub/assets//content', AssetContent.as_view()), path('workspace//ai_hub/parsers//test', ParserTest.as_view()), path('workspace//ai_hub/published//run', PublishedRun.as_view()), path('workspace//ai_hub/', HubView.as_view()), diff --git a/apps/system_manage/views/ai_hub.py b/apps/system_manage/views/ai_hub.py new file mode 100644 index 000000000..43ae45d82 --- /dev/null +++ b/apps/system_manage/views/ai_hub.py @@ -0,0 +1,201 @@ +import hashlib +import secrets +from django.core.cache import cache +from django.db import transaction +from django.utils import timezone +from rest_framework.views import APIView +from rest_framework.exceptions import ValidationError, PermissionDenied +from common.auth import TokenAuth +from common.result import result +from system_manage import ai_hub as hub +from system_manage.models.ai_hub import AIService, AITask + + +class HubView(APIView): + authentication_classes = [TokenAuth] + + def initial(self, request, *args, **kwargs): + super().initial(request, *args, **kwargs) + from users.serializers.user import get_workspace_list_by_user + if not request.user.is_active or kwargs['workspace_id'] not in {str(w['id']) for w in get_workspace_list_by_user(request.user.id)}: + raise PermissionDenied('无权访问') + + def get(self, request, workspace_id, kind, pk=None): + if kind == 'resources': + parsers = hub.visible(hub.AIParserResource.objects.filter(workspace_id=workspace_id, enabled=True), request.user) + from users.models import User + from users.models.user_manage_ext import UserActivationApplication + pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True) + return result.success({'parsers': [hub.serialize(p) for p in parsers], + 'models': list(hub.authorized_models(workspace_id, request.user).values('id', 'name', 'model_type')), + 'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if hub.admin(request.user) else [], + 'is_admin': hub.admin(request.user)}) + if pk: + obj = hub.one(kind, pk, workspace_id, request.user) + data = hub.serialize(obj, detail=True) + if kind == 'parsers' and hub.admin(request.user): + data['endpoint'] = obj.endpoint + return result.success(data) + if kind not in hub.KINDS: + raise ValidationError('资源类型无效') + query = hub.visible(hub.KINDS[kind].objects.filter(workspace_id=workspace_id), request.user) + if kind == 'tasks': + query = query.defer('file_data', 'input_data', 'snapshot', 'result') + elif kind == 'artifacts': + query = query.defer('original', 'markdown', 'history') + return result.success([hub.serialize(obj) for obj in query.order_by('-created_at')[:200]]) + + def post(self, request, workspace_id, kind, pk=None): + data = request.data + if request.FILES: + import json + try: + data = json.loads(request.data.get('data', '{}')) + except (ValueError, TypeError): + raise ValidationError('任务参数无效') + if not isinstance(data, dict): + raise ValidationError('请求参数无效') + if pk: + with transaction.atomic(): + obj = hub.one(kind, pk, workspace_id, request.user, edit=True) + if kind == 'parsers': + obj = hub.parser_save(workspace_id, request.user, data, obj) + elif kind == 'artifacts': + obj = hub.artifact_save(workspace_id, request.user, data, obj) + elif kind == 'services': + hub.require_admin(request.user) + obj = AIService.objects.select_for_update().get(pk=obj.pk) + if 'name' in data: + obj.name = hub.text(data['name'], '名称', limit=128) + if 'requests_per_minute' in data: + obj.requests_per_minute = hub.integer(data['requests_per_minute'], 1, 600, '调用频率') + if 'enabled' in data: + if not isinstance(data['enabled'], bool): + raise ValidationError('启用状态无效') + obj.enabled = data['enabled'] + if 'allowed_users' in data: + obj.allowed_users = hub.users(data['allowed_users']) + key = secrets.token_urlsafe(32) if data.get('rotate_key') else None + if key: + obj.key_hash = hashlib.sha256(key.encode()).hexdigest() + if data.get('revoke_key'): + obj.key_hash = '' + obj.save() + output = hub.serialize(obj) + if key: + output['api_key'] = key + return result.success(output) + elif kind == 'tasks': + obj = AITask.objects.select_for_update().get(pk=obj.pk) + if obj.status != 'FAILED': + raise ValidationError('仅失败任务可重试') + if hub.make_snapshot(workspace_id, request.user, obj.snapshot['nodes']) != obj.snapshot['nodes']: + raise ValidationError('资源版本已变更,请重新提交任务') + if obj.service_id and not obj.service.enabled: + raise ValidationError('服务已停用') + obj.status, obj.error, obj.steps = 'PENDING', '', [] + obj.attempt += 1 + obj.started_at = obj.finished_at = None + obj.save() + else: + raise ValidationError('流程版本不可修改,请另存版本') + return result.success(hub.serialize(obj, detail=True)) + if kind == 'parsers': + obj = hub.parser_save(workspace_id, request.user, data) + elif kind == 'flows': + obj = hub.flow_create(workspace_id, request.user, data) + elif kind == 'services': + obj = hub.service_create(workspace_id, request.user, data) + elif kind == 'tasks': + # Bound pending backlog per user; GPU work is consumed by the isolated worker. + if AITask.objects.filter(owner_id=request.user.id, status__in=['PENDING', 'RUNNING']).count() >= 10: + raise ValidationError('待处理任务已达上限') + obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file')) + elif kind == 'artifacts': + obj = hub.artifact_save(workspace_id, request.user, data) + else: + raise ValidationError('资源类型无效') + return result.success(hub.serialize(obj, detail=True)) + + +class ParserTest(HubView): + http_method_names = ['post', 'options'] + + def post(self, request, workspace_id, pk): + hub.require_admin(request.user) + parser = hub.one('parsers', pk, workspace_id, request.user) + return result.success(hub.parser_test(parser)) + + +class AssetContent(HubView): + http_method_names = ['get', 'options'] + + def get(self, request, workspace_id, pk): + from django.http import HttpResponse + from system_manage.ai_hub_assets import image_bytes + content, meta = image_bytes(workspace_id, request.user, pk) + response = HttpResponse(content, content_type=meta['mime']) + response['Cache-Control'] = 'private, no-store' + response['X-Content-Type-Options'] = 'nosniff' + return response + + +class MarkdownExport(HubView): + http_method_names = ['post', 'options'] + + def post(self, request, workspace_id): + import io + import zipfile + from django.http import HttpResponse + from system_manage.ai_hub_assets import ASSET, asset_lock, image_bytes, normalize + markdown = hub.text(request.data.get('markdown'), '文档内容') + with asset_lock(): + markdown = normalize(markdown, workspace_id, request.user) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive: + replacements = {} + total = 0 + for match in ASSET.finditer(markdown): + if match[0] in replacements: + continue + content, meta = image_bytes(workspace_id, request.user, match[2]) + total += len(content) + if total > 100 * 1024 * 1024: + raise ValidationError('导出图片总大小超过限制') + extension = meta['mime'].split('/')[1] + name = f'images/{match[2]}.{extension}' + archive.writestr(name, content) + replacements[match[0]] = name + for url, name in replacements.items(): + markdown = markdown.replace(url, name) + archive.writestr('document.md', markdown) + response = HttpResponse(buffer.getvalue(), content_type='application/zip') + response['Content-Disposition'] = 'attachment; filename="markdown.zip"' + return response + + +class PublishedRun(HubView): + """Internal service: login AND explicit resource grants; Key is an optional extra credential.""" + authentication_classes = [TokenAuth] + http_method_names = ['post', 'options'] + + def post(self, request, workspace_id, pk): + service = hub.one('services', pk, workspace_id, request.user) + if service.key_hash: + supplied = hashlib.sha256(request.headers.get('X-Service-Key', '').encode()).hexdigest() + if not secrets.compare_digest(supplied, service.key_hash): + raise PermissionDenied('服务Key无效') + minute = int(timezone.now().timestamp()) // 60 + key = f'ai-hub-rate:{service.id}:{minute}' + cache.add(key, 0, 70) + if cache.incr(key) > service.requests_per_minute: + raise ValidationError('调用过于频繁') + data = request.data + if request.FILES: + import json + try: + data = json.loads(request.data.get('data', '{}')) + except (ValueError, TypeError): + raise ValidationError('任务参数无效') + task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service) + return result.success(hub.serialize(task)) diff --git a/apps/users/migrations/0002_user_activation_and_role_assignment.py b/apps/users/migrations/0002_user_activation_and_role_assignment.py new file mode 100644 index 000000000..9b558a373 --- /dev/null +++ b/apps/users/migrations/0002_user_activation_and_role_assignment.py @@ -0,0 +1,48 @@ +# Generated for user activation audit and role assignment +import uuid_utils.compat +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='UserActivationApplication', + fields=[ + ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), + ('email', models.EmailField(blank=True, default='', max_length=254, verbose_name='注册邮箱')), + ('email_verified', models.BooleanField(default=False, verbose_name='邮箱是否已验证')), + ('status', models.CharField(choices=[('PENDING', '待审核'), ('APPROVED', '已通过'), ('REJECTED', '已拒绝')], db_index=True, default='PENDING', max_length=20, verbose_name='审核状态')), + ('applied_at', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='申请时间')), + ('audit_time', models.DateTimeField(blank=True, null=True, verbose_name='审核时间')), + ('reject_reason', models.CharField(blank=True, default='', max_length=500, verbose_name='拒绝原因')), + ('audit_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='审核人')), + ('user', models.OneToOneField(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='activation_application', to='users.user', verbose_name='关联用户')), + ], + options={ + 'db_table': 'user_activation_application', + 'ordering': ['-applied_at'], + }, + ), + migrations.CreateModel( + name='UserRoleAssignment', + fields=[ + ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')), + ('role_id', models.CharField(db_index=True, default='USER', max_length=64, verbose_name='角色标识')), + ('workspace_id', models.CharField(db_index=True, default='default', max_length=128, verbose_name='工作空间id')), + ('granted_at', models.DateTimeField(auto_now_add=True, verbose_name='授权时间')), + ('granted_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='授权人')), + ('user', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='role_assignments', to='users.user', verbose_name='用户')), + ], + options={ + 'db_table': 'user_role_assignment', + 'ordering': ['granted_at'], + 'unique_together': {('user', 'role_id', 'workspace_id')}, + }, + ), + ] diff --git a/apps/users/models/user_manage_ext.py b/apps/users/models/user_manage_ext.py new file mode 100644 index 000000000..405c9a1e1 --- /dev/null +++ b/apps/users/models/user_manage_ext.py @@ -0,0 +1,57 @@ +# coding=utf-8 +""" +用户管理扩展模型: +1. UserActivationApplication:初次注册用户的待激活审核申请(独立数据对象) +2. UserRoleAssignment:用户-角色-工作空间授权关系(独立数据对象) + +说明:这两个模型用于规格书 2.2.2「设置初次注册用户为激活用户」「更改用户角色」 +两个功能过程的独立申报,避免与「启用/停用用户账号」「更改用户信息」共用同一数据对象。 +""" +import uuid_utils.compat as uuid + +from django.db import models + + +class ActivationStatus(models.TextChoices): + PENDING = "PENDING", "待审核" + APPROVED = "APPROVED", "已通过" + REJECTED = "REJECTED", "已拒绝" + + +class UserActivationApplication(models.Model): + """初次注册用户的激活审核申请。""" + + id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id") + user = models.OneToOneField('users.User', on_delete=models.CASCADE, db_constraint=False, + related_name="activation_application", verbose_name="关联用户") + email = models.EmailField(verbose_name="注册邮箱", blank=True, default="") + email_verified = models.BooleanField(default=False, verbose_name="邮箱是否已验证") + status = models.CharField(max_length=20, choices=ActivationStatus.choices, + default=ActivationStatus.PENDING, db_index=True, verbose_name="审核状态") + applied_at = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="申请时间") + audit_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False, + null=True, blank=True, related_name="+", verbose_name="审核人") + audit_time = models.DateTimeField(null=True, blank=True, verbose_name="审核时间") + reject_reason = models.CharField(max_length=500, blank=True, default="", verbose_name="拒绝原因") + + class Meta: + db_table = "user_activation_application" + ordering = ["-applied_at"] + + +class UserRoleAssignment(models.Model): + """用户-角色-工作空间授权关系。""" + + id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id") + user = models.ForeignKey('users.User', on_delete=models.CASCADE, db_constraint=False, + related_name="role_assignments", verbose_name="用户") + role_id = models.CharField(max_length=64, default="USER", db_index=True, verbose_name="角色标识") + workspace_id = models.CharField(max_length=128, default="default", db_index=True, verbose_name="工作空间id") + granted_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False, + null=True, blank=True, related_name="+", verbose_name="授权人") + granted_at = models.DateTimeField(auto_now_add=True, verbose_name="授权时间") + + class Meta: + db_table = "user_role_assignment" + unique_together = ("user", "role_id", "workspace_id") + ordering = ["granted_at"] diff --git a/apps/users/serializers/register.py b/apps/users/serializers/register.py new file mode 100644 index 000000000..160ffa938 --- /dev/null +++ b/apps/users/serializers/register.py @@ -0,0 +1,113 @@ +import json + +from rest_framework import serializers +from django.db import IntegrityError, transaction +from django.db.models import Q +from django.utils import timezone + +from common.exception.app_exception import AppApiException +from common.utils.common import password_encrypt +from common.utils.rsa_util import decrypt +from users.models import User, UserActivationApplication, ActivationStatus +from users.serializers.user import UserManageSerializer + + +class RegisterSerializer(UserManageSerializer.UserInstance): + re_password = serializers.CharField(required=True, write_only=True, trim_whitespace=False) + + def get_fields(self): + fields = super().get_fields() + for name in ('source', 'pending_activation'): + fields.pop(name) + return fields + + def is_valid(self, *, raise_exception=True): + # 重复账号的处理由 register 在锁定申请后按审核状态判断。 + return serializers.Serializer.is_valid(self, raise_exception=raise_exception) + + def validate(self, attrs): + if attrs['password'] != attrs['re_password']: + raise serializers.ValidationError({'re_password': '两次密码不一致'}) + return attrs + + @staticmethod + def _match_condition(username: str, email: str) -> Q: + condition = Q(username=username) + if email: + condition = condition | Q(email=email) + return condition + + @staticmethod + def _re_register(existing_user: User, data: dict) -> dict: + """被拒绝的用户修改注册信息后重新提交:清空拒绝原因,回到待审核。""" + condition = RegisterSerializer._match_condition(data['username'], data.get('email', '')) + condition |= Q(nick_name=data['nick_name']) + conflict = User.objects.filter(condition).exclude(id=existing_user.id).exists() + if conflict: + raise AppApiException(500, '用户名、姓名或邮箱已被其他账号占用') + with transaction.atomic(): + existing_user.username = data['username'] + existing_user.nick_name = data.get('nick_name', '') or existing_user.nick_name + if data.get('email'): + existing_user.email = data['email'] + existing_user.phone = data.get('phone', '') or '' + existing_user.password = password_encrypt(data['password']) + existing_user.is_active = False + existing_user.save() + # applied_at 是 auto_now_add,只能用 update 直接改库刷新申请时间 + UserActivationApplication.objects.filter(id=existing_user.activation_application.id).update( + status=ActivationStatus.PENDING, + reject_reason='', + email=existing_user.email or '', + email_verified=False, + audit_by=None, + audit_time=None, + applied_at=timezone.now(), + ) + return {'status': ActivationStatus.PENDING, 're_register': True} + + @staticmethod + @transaction.atomic + def register(instance): + payload = dict(instance) + if payload.get('encryptedData'): + try: + credentials = json.loads(decrypt(payload['encryptedData'])) + payload['password'] = credentials['password'] + payload['re_password'] = credentials['re_password'] + except (ValueError, TypeError, KeyError): + raise AppApiException(500, '注册信息无效') + serializer = RegisterSerializer(data=payload) + serializer.is_valid(raise_exception=True) + data = {key: serializer.validated_data.get(key, '') for key in + ('username', 'nick_name', 'email', 'phone', 'password')} + # Public registration never accepts account state, source or role assignments. + data.update(pending_activation=True, source='LOCAL') + + existing = User.objects.select_for_update().filter( + RegisterSerializer._match_condition(data['username'], data.get('email', '')) + ).first() + if existing is not None: + application = UserActivationApplication.objects.select_for_update().filter(user=existing).first() + if application is None: + raise AppApiException(500, '该用户名或邮箱已存在,请直接登录或联系管理员') + if application.status == ActivationStatus.PENDING: + raise AppApiException(500, '注册申请审核中,请等待管理员审核,请勿重复提交') + if application.status == ActivationStatus.APPROVED: + raise AppApiException(500, '该账号已激活,请直接登录') + if application.status != ActivationStatus.REJECTED: + raise AppApiException(500, '该申请不允许重新提交') + # 公开注册入口不能凭用户名或邮箱覆盖别人的被拒绝账号。 + if (existing.username != data['username'] or existing.source != 'LOCAL' + or existing.password != password_encrypt(data['password'])): + raise AppApiException(500, '重新提交请使用原用户名和密码') + # 仅“已拒绝”的申请允许修改注册信息后重新提交 + return RegisterSerializer._re_register(existing, data) + + if User.objects.filter(nick_name=data['nick_name']).exists(): + raise AppApiException(500, '姓名已存在') + try: + UserManageSerializer().save(data, user_id=None, with_valid=False) + except IntegrityError: + raise AppApiException(500, '用户名、姓名或邮箱已存在') + return {'status': ActivationStatus.PENDING} diff --git a/apps/users/serializers/user_manage_ext.py b/apps/users/serializers/user_manage_ext.py new file mode 100644 index 000000000..1aa1b6e06 --- /dev/null +++ b/apps/users/serializers/user_manage_ext.py @@ -0,0 +1,224 @@ +# coding=utf-8 +"""用户管理扩展序列化器:待激活审核、角色授权。""" +from django.core.cache import cache +from django.core.mail import send_mail +from django.core.mail.backends.smtp import EmailBackend +from django.db import transaction +from django.db.models import Q +from django.utils import timezone +from rest_framework import serializers + +from common.constants.cache_version import Cache_Version +from common.constants.permission_constants import RoleConstants +from common.exception.app_exception import AppApiException +from common.utils.logger import maxkb_logger +from system_manage.models import SystemSetting, SettingType +from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment +from users.serializers.user import update_user_role + +ROLE_OPTIONS = [ + {"id": RoleConstants.USER.name, "name": "普通用户"}, + {"id": RoleConstants.ADMIN.name, "name": "管理员"}, +] +VALID_ROLE_IDS = {item["id"] for item in ROLE_OPTIONS} + + +def _fmt(value): + if not value: + return "" + try: + value = timezone.localtime(value) + except Exception: + pass + return value.strftime("%Y-%m-%d %H:%M:%S") + + +def invalidate_user_auth_cache(user_id): + """角色/状态变化后清掉权限与角色缓存,下一次请求重新计算。""" + for cache_version in (Cache_Version.PERMISSION_LIST, Cache_Version.ROLE_LIST, Cache_Version.USER): + cache.delete(cache_version.get_key(user_id=user_id), version=cache_version.get_version()) + + +def send_activation_notice(user, approved: bool, reason: str = "") -> bool: + """激活审核结果邮件通知。 + + 仅在系统设置中配置了发件邮箱(SystemSetting EMAIL)且用户填了邮箱时发送; + 发送失败只记录日志,不影响审核结果。 + """ + try: + if not user.email: + return False + setting = SystemSetting.objects.filter(type=SettingType.EMAIL.value).first() + meta = (setting.meta or {}) if setting else {} + if not meta.get("email_host"): + return False + name = user.nick_name or user.username + if approved: + subject = "【AI-RAG 知识库平台】账号激活审核通过" + body = (f"您好 {name}:\n\n" + f"您提交的注册申请已审核通过,账号已激活,现在可以登录平台使用。\n\n" + f"如非本人操作,请忽略本邮件。") + else: + subject = "【AI-RAG 知识库平台】账号激活审核未通过" + body = (f"您好 {name}:\n\n" + f"很抱歉,您提交的注册申请未通过审核。\n" + f"拒绝原因:{reason or '不符合激活条件'}\n\n" + f"请在注册页面使用原用户名和密码修改并重新提交资料,审核通过后即可登录。") + connection = EmailBackend(meta.get("email_host"), meta.get("email_port"), + meta.get("email_host_user"), meta.get("email_host_password"), + meta.get("email_use_tls"), False, meta.get("email_use_ssl"), timeout=10) + send_mail(subject, body, + from_email=meta.get("from_email") or meta.get("email_host_user"), + recipient_list=[user.email], + fail_silently=False, connection=connection) + return True + except Exception as exc: + maxkb_logger.error(f"send activation notice failed: user={getattr(user, 'id', '')} {exc}") + return False + + +class ActivationSerializer(serializers.Serializer): + keyword = serializers.CharField(required=False, allow_blank=True, default="") + status = serializers.CharField(required=False, allow_blank=True, default=ActivationStatus.PENDING) + + @staticmethod + def page(current_page: int, page_size: int, keyword: str = "", status: str = ActivationStatus.PENDING): + query_set = UserActivationApplication.objects.select_related("user").all() + if status == 'UNAPPROVED': + query_set = query_set.filter(status__in=[ActivationStatus.PENDING, ActivationStatus.REJECTED]) + elif status and status != "ALL": + query_set = query_set.filter(status=status) + if keyword: + query_set = query_set.filter( + Q(user__username__icontains=keyword) + | Q(user__nick_name__icontains=keyword) + | Q(user__email__icontains=keyword) + | Q(user__phone__icontains=keyword) + ) + total = query_set.count() + start = max(0, (current_page - 1) * page_size) + records = [] + for application in query_set.order_by("-applied_at")[start:start + page_size]: + user = application.user + records.append({ + "id": str(application.id), + "user_id": str(user.id), + "username": user.username, + "nick_name": user.nick_name, + "email": user.email or application.email or "", + "phone": user.phone or '', + "email_verified": application.email_verified, + "status": application.status, + "applied_at": _fmt(application.applied_at), + "audit_time": _fmt(application.audit_time), + "reject_reason": application.reject_reason, + }) + return {"records": records, "total": total, "current_page": current_page, "page_size": page_size} + + @staticmethod + @transaction.atomic + def approve(application_id: str, audit_user_id: str, role_id: str = RoleConstants.USER.name, + workspace_id: str = "default"): + application = (UserActivationApplication.objects.select_for_update() + .select_related("user").filter(id=application_id).first()) + if application is None: + raise AppApiException(500, "待激活申请不存在") + if application.status != ActivationStatus.PENDING: + raise AppApiException(500, "该申请已处理,不能重复审核") + if role_id not in VALID_ROLE_IDS: + raise AppApiException(500, "角色不合法") + application.status = ActivationStatus.APPROVED + application.reject_reason = "" + application.audit_by_id = audit_user_id + application.audit_time = timezone.now() + application.save() + + user = application.user + user.is_active = True + user.role = role_id + user.save() + + # 审核通过时同步写入角色授权关系,形成独立的授权数据对象 + UserRoleAssignment.objects.filter(user_id=user.id).delete() + UserRoleAssignment.objects.create( + user_id=user.id, role_id=role_id, + workspace_id=workspace_id or "default", granted_by_id=audit_user_id + ) + invalidate_user_auth_cache(user.id) + transaction.on_commit(lambda: send_activation_notice(user, True)) + return {"id": str(application.id), "status": application.status, "user_id": str(user.id)} + + @staticmethod + @transaction.atomic + def reject(application_id: str, audit_user_id: str, reason: str = ""): + application = (UserActivationApplication.objects.select_for_update() + .select_related("user").filter(id=application_id).first()) + if application is None: + raise AppApiException(500, "待激活申请不存在") + if application.status != ActivationStatus.PENDING: + raise AppApiException(500, "该申请已处理,不能重复审核") + application.status = ActivationStatus.REJECTED + application.audit_by_id = audit_user_id + application.audit_time = timezone.now() + application.reject_reason = reason or "不符合激活条件" + application.save() + application.user.is_active = False + application.user.save() + invalidate_user_auth_cache(application.user.id) + transaction.on_commit(lambda: send_activation_notice(application.user, False, application.reject_reason)) + return {"id": str(application.id), "status": application.status, "user_id": str(application.user_id)} + + +class RoleAssignmentSerializer(serializers.Serializer): + role_id = serializers.CharField(required=True) + workspace_ids = serializers.ListField(child=serializers.CharField(), required=False, allow_empty=True) + + @staticmethod + def detail(user_id: str): + user = User.objects.filter(id=user_id).first() + if user is None: + raise AppApiException(500, "用户不存在") + assignments = UserRoleAssignment.objects.filter(user_id=user_id).order_by("granted_at") + records = [{ + "role_id": item.role_id, + "workspace_id": item.workspace_id, + "granted_at": _fmt(item.granted_at), + } for item in assignments] + if not records: + records = [{"role_id": user.role, "workspace_id": "default", "granted_at": ""}] + return { + "user": {"id": str(user.id), "username": user.username, "nick_name": user.nick_name, + "email": user.email or "", "role": user.role, "is_active": user.is_active}, + "assignments": records, + "available_roles": ROLE_OPTIONS, + } + + @staticmethod + @transaction.atomic + def save(user_id: str, role_settings, granted_by_id: str): + user = User.objects.select_for_update().filter(id=user_id).first() + if user is None: + raise AppApiException(500, "用户不存在") + if not isinstance(role_settings, list) or len(role_settings) != 1 or not isinstance(role_settings[0], dict): + raise AppApiException(500, "请选择一个角色") + item = role_settings[0] + role_id = item.get("role_id") + if role_id not in VALID_ROLE_IDS: + raise AppApiException(500, "角色只能为普通用户或管理员") + if item.get('workspace_ids') not in (None, [], ['default']): + raise AppApiException(500, '角色配置无效') + if str(user.id) == 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab': + if role_id != RoleConstants.ADMIN.name: + raise AppApiException(500, '不能修改内置管理员角色') + return RoleAssignmentSerializer.detail(str(user.id)) + UserRoleAssignment.objects.filter(user_id=user.id).delete() + UserRoleAssignment.objects.create( + user_id=user.id, role_id=role_id, workspace_id='default', granted_by_id=granted_by_id + ) + user.role = role_id + user.save() + update_user_role({'role_setting': [{'role_id': role_id, 'workspace_ids': [ + 'None' if role_id == RoleConstants.ADMIN.name else 'default' + ]}]}, user, granted_by_id) + invalidate_user_auth_cache(user.id) + return RoleAssignmentSerializer.detail(str(user.id)) diff --git a/apps/users/views/register.py b/apps/users/views/register.py new file mode 100644 index 000000000..cc80bb4c5 --- /dev/null +++ b/apps/users/views/register.py @@ -0,0 +1,15 @@ +from drf_spectacular.utils import extend_schema +from rest_framework.permissions import AllowAny +from rest_framework.views import APIView + +from common.result import result +from users.serializers.register import RegisterSerializer + + +class RegisterView(APIView): + authentication_classes = [] + permission_classes = [AllowAny] + + @extend_schema(summary='用户注册', tags=['User Management']) + def post(self, request): + return result.success(RegisterSerializer.register(request.data)) diff --git a/apps/users/views/user_manage_ext.py b/apps/users/views/user_manage_ext.py new file mode 100644 index 000000000..b84793645 --- /dev/null +++ b/apps/users/views/user_manage_ext.py @@ -0,0 +1,73 @@ +# coding=utf-8 +"""用户管理扩展视图:待激活审核、角色授权。""" +from drf_spectacular.utils import extend_schema +from rest_framework.request import Request +from rest_framework.views import APIView + +from common.auth.authenticate import TokenAuth +from common.auth.authentication import has_permissions +from common.constants.permission_constants import PermissionConstants, RoleConstants +from common.log.log import log +from common.result import result +from users.models import ActivationStatus +from users.serializers.user_manage_ext import ActivationSerializer, RoleAssignmentSerializer + + +class UserActivationPendingView(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="待激活注册申请列表", tags=["User Management"]) + @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + def get(self, request: Request): + current_page = int(request.query_params.get("current_page", 1)) + page_size = int(request.query_params.get("page_size", 20)) + keyword = request.query_params.get("keyword", "") + status = request.query_params.get("status", ActivationStatus.PENDING) + return result.success(ActivationSerializer.page(current_page, page_size, keyword, status)) + + +class UserActivationApproveView(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['POST'], summary="通过待激活注册申请", tags=["User Management"]) + @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @log(menu='User management', operate='Approve user activation', + get_operation_object=lambda r, k: {'name': k.get('application_id')}) + def post(self, request: Request, application_id: str): + role_id = request.data.get("role_id", RoleConstants.USER.name) + workspace_id = request.data.get("workspace_id", "default") + return result.success( + ActivationSerializer.approve(application_id, str(request.user.id), role_id, workspace_id) + ) + + +class UserActivationRejectView(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['POST'], summary="拒绝待激活注册申请", tags=["User Management"]) + @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @log(menu='User management', operate='Reject user activation', + get_operation_object=lambda r, k: {'name': k.get('application_id')}) + def post(self, request: Request, application_id: str): + reason = request.data.get("reason", "") + return result.success( + ActivationSerializer.reject(application_id, str(request.user.id), reason) + ) + + +class UserRoleAssignmentView(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="查看用户角色授权", tags=["User Management"]) + @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + def get(self, request: Request, user_id: str): + return result.success(RoleAssignmentSerializer.detail(user_id)) + + @extend_schema(methods=['PUT'], summary="分配用户角色", tags=["User Management"]) + @has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN) + @log(menu='User management', operate='Assign user role', + get_operation_object=lambda r, k: {'name': k.get('user_id')}) + def put(self, request: Request, user_id: str): + return result.success( + RoleAssignmentSerializer.save(user_id, request.data.get("role_settings", []), str(request.user.id)) + ) diff --git a/scripts/README_ai_hub_assets.md b/scripts/README_ai_hub_assets.md new file mode 100644 index 000000000..dbb17b198 --- /dev/null +++ b/scripts/README_ai_hub_assets.md @@ -0,0 +1,20 @@ +# 思维导图图片资源 + +解析任务、手工提交和保存图谱时,将支持的 Base64 位图转换为私有图片文件,Markdown 保存稳定的 `/admin/api/workspace/.../ai_hub/assets/.../content` 地址。历史图谱在再次保存时转换原稿和历史版本中的内联图片。 + +默认持久化目录为项目根目录 `data/ai-hub-assets`;可通过环境变量 `AI_HUB_ASSET_DIR` 指定。Web 与独立 Hub Worker 必须使用相同目录,多机部署需挂载共享存储。备份应同时包含现有数据库与此目录。增量发布不可覆盖该目录。 + +文件以工作空间、用户和内容哈希去重,旁路 JSON 保存归属、格式、大小和最后引用时间。无需数据库迁移。图片接口使用现有用户和工作空间鉴权,父页面通过二进制通信供导图预览使用,令牌不进入图片 URL。浏览器仅在当前页面内缓存图片,图片展示可点击放大,失败后可重试。 + +单图上限 20 MiB、4000 万像素;单次转换图片合计上限 100 MiB。支持 PNG、JPEG、GIF、WebP、BMP。图谱及其原稿、历史版本所引用的图片长期保留;运行中或待处理任务中的图片不会清理。其余资源最后引用七天后可清理,未保存的旧任务图片届时可能失效。 + +已有独立 Hub Worker 每运行 24 小时执行一次清理,不新增定时进程。手工核验可在原有 Web 环境中执行: + +```sh +venv/bin/python scripts/ai_hub_ops.py cleanup-assets +venv/bin/python scripts/ai_hub_ops.py cleanup-assets --apply +``` + +首条命令只报告数量。清理范围仅限本模块资源目录,不涉及原文档、数据库记录、模型或解析服务文件。 + +含资源图片的 Markdown 下载为 ZIP,包含 `document.md` 和 `images/`;HTML、导图 SVG 导出会嵌入图片,以便离线打开。智能整理使用图片占位符;重复占位符只恢复一次,遗漏的图片自动补入“原文图片”部分,解析原稿始终保留。 diff --git a/scripts/ai_hub_ops.py b/scripts/ai_hub_ops.py new file mode 100644 index 000000000..906432eb0 --- /dev/null +++ b/scripts/ai_hub_ops.py @@ -0,0 +1,47 @@ +"""Explicit AI hub operations; defaults to a read-only schema check, never auto-migrates.""" +import argparse +import os +from pathlib import Path +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['check', 'plan', 'migrate', 'worker', 'cleanup-assets'], nargs='?', default='check') + parser.add_argument('--apply', action='store_true', help='Apply expired image cleanup (default: report only)') + parser.add_argument('--once', action='store_true') + parser.add_argument('--recover', action='store_true') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + os.chdir(root) + sys.path.insert(0, str(root / 'apps')) + os.environ['SERVER_NAME'] = 'web' + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + from django.conf import settings + if settings.LANGUAGE_CODE.lower() == 'zh-cn': + settings.LANGUAGE_CODE = 'zh-hans' + import django + django.setup() + from django.core.management import call_command + from django.db import connection + from django.db.migrations.executor import MigrationExecutor + target = [('system_manage', '0007_ai_hub')] + if args.action in ('plan', 'migrate'): + executor = MigrationExecutor(connection) + plan = executor.migration_plan(target) + if any(backwards for _, backwards in plan): + raise RuntimeError('存在更新迁移,不允许回退,请先核对版本') + call_command('migrate', 'system_manage', '0007_ai_hub', plan=args.action == 'plan', interactive=False) + elif args.action == 'cleanup-assets': + call_command('ai_hub_cleanup_assets', apply=args.apply) + elif args.action == 'worker': + call_command('ai_hub_worker', once=args.once, recover=args.recover) + else: + from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact + tables = set(connection.introspection.table_names()) + for model in (AIParserResource, AIFlow, AIService, AITask, AIArtifact): + print(f'{model._meta.db_table}: {"ready" if model._meta.db_table in tables else "missing"}') + + +if __name__ == '__main__': + main() diff --git a/scripts/check_ai_hub.py b/scripts/check_ai_hub.py new file mode 100644 index 000000000..454ef5b9e --- /dev/null +++ b/scripts/check_ai_hub.py @@ -0,0 +1,38 @@ +"""Isolated SQLite verification; never connects to the shared PostgreSQL or Redis.""" +import importlib +import os +from pathlib import Path +import sys +import unittest + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root / 'apps')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') +from django.apps import AppConfig +from django.conf import settings +settings.LOGGING_CONFIG = None +settings.USE_I18N = False +settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} +settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in ( + 'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common', + 'system_manage', 'models_provider', 'application', 'chat', 'oss')] +import django +django.setup() +if '--migration' in sys.argv or '--migration-check' in sys.argv: + from django.core.management import call_command + call_command('makemigrations', 'system_manage', name='ai_hub', + check='--migration-check' in sys.argv, dry_run='--migration-check' in sys.argv) + sys.exit(0) +from django.db import connection +from users.models import User +from users.models.user_manage_ext import UserActivationApplication +from models_provider.models import Model +from knowledge.models import KnowledgeFolder, Knowledge +from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact +with connection.schema_editor() as editor: + for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge, AIParserResource, AIFlow, AIService, AITask, AIArtifact): + editor.create_model(model) +suite = unittest.defaultTestLoader.loadTestsFromName('system_manage.test_ai_hub') +outcome = unittest.TextTestRunner(verbosity=2).run(suite) +sys.exit(0 if outcome.wasSuccessful() else 1) diff --git a/scripts/check_knowledge_ext.py b/scripts/check_knowledge_ext.py new file mode 100644 index 000000000..f168d2105 --- /dev/null +++ b/scripts/check_knowledge_ext.py @@ -0,0 +1,36 @@ +"""Run knowledge metadata and segmentation regression tests against an isolated in-memory database.""" +import importlib +import os +from pathlib import Path +import sys +import unittest + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root / 'apps')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + +from django.apps import AppConfig +from django.conf import settings + +settings.LOGGING_CONFIG = None +settings.USE_I18N = False +settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} +settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in ( + 'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common', + 'system_manage', 'models_provider', 'application', 'chat', 'oss', +)] + +import django +django.setup() + +from django.db import connection +from users.models import User, UserActivationApplication +from models_provider.models import Model +from knowledge.models import KnowledgeFolder, Knowledge +with connection.schema_editor() as editor: + for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge): + editor.create_model(model) +suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_knowledge_ext') +result = unittest.TextTestRunner(verbosity=2).run(suite) +sys.exit(0 if result.wasSuccessful() else 1) diff --git a/scripts/check_knowledge_rebuild.py b/scripts/check_knowledge_rebuild.py new file mode 100644 index 000000000..a40b2f3e2 --- /dev/null +++ b/scripts/check_knowledge_rebuild.py @@ -0,0 +1,35 @@ +"""Run independent index rebuild regression tests against an isolated in-memory database.""" +import importlib +import os +from pathlib import Path +import sys +import unittest + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root / 'apps')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + +from django.apps import AppConfig +from django.conf import settings + +settings.LOGGING_CONFIG = None +settings.USE_I18N = False +settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} +settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in ( + 'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common', + 'system_manage', 'models_provider', 'application', 'chat', 'oss', +)] + +import django +django.setup() + +from django.db import connection +from knowledge.models import KnowledgeRebuildTask + +with connection.schema_editor() as editor: + editor.create_model(KnowledgeRebuildTask) + +suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_rebuild') +result = unittest.TextTestRunner(verbosity=2).run(suite) +sys.exit(0 if result.wasSuccessful() else 1) diff --git a/scripts/check_local_runtime_api.py b/scripts/check_local_runtime_api.py new file mode 100644 index 000000000..cd51cec43 --- /dev/null +++ b/scripts/check_local_runtime_api.py @@ -0,0 +1,32 @@ +"""Exercise the signed API under the actual minimal model-service app configuration.""" +import os +from pathlib import Path +import sys +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'apps')) +os.environ['SERVER_NAME'] = 'local_model' +os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0' +from django.conf import settings +settings.configure(SECRET_KEY='test-only', AUTH_HANDLES=[], USE_I18N=False, + INSTALLED_APPS=['django.contrib.contenttypes', 'local_model'], + DATABASES={'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}) +import django +django.setup() +from rest_framework.test import APIRequestFactory +from common.model_runtime_client import signed_request +from local_model.views.runtime import LocalModelRuntime +factory = APIRequestFactory() +with patch('common.model_runtime_client._secret', return_value=b'test-only'), patch('local_model.views.runtime.manager') as manager: + manager.status.return_value = {'status': 'UNLOADED'} + body, signature = signed_request({'operation': 'status', 'model_id': '00000000-0000-4000-8000-000000000001'}) + response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body, + content_type='application/json', HTTP_X_MODEL_RUNTIME_SIGNATURE=signature)) + assert response.status_code == 200, response.content + manager.status.assert_called_once() + manager.reset_mock() + response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body, content_type='application/json')) + assert response.status_code == 403, response.status_code + manager.status.assert_not_called() +print('Minimal model-service API: signed request accepted; unsigned request rejected') diff --git a/scripts/check_model_loading.py b/scripts/check_model_loading.py new file mode 100644 index 000000000..08439a2d9 --- /dev/null +++ b/scripts/check_model_loading.py @@ -0,0 +1,6 @@ +"""Model loading regressions against SQLite in memory; never connects to shared services.""" +from pathlib import Path + +source = (Path(__file__).with_name('check_knowledge_ext.py')).read_text(encoding='utf-8') +source = source.replace("loadTestsFromName('knowledge.test_knowledge_ext')", "loadTestsFromNames(['models_provider.test_model_runtime', 'models_provider.test_model_loading_api', 'knowledge.test_knowledge_ext'])") +exec(compile(source, __file__, 'exec')) diff --git a/scripts/check_user_registration.py b/scripts/check_user_registration.py new file mode 100644 index 000000000..2dcb40697 --- /dev/null +++ b/scripts/check_user_registration.py @@ -0,0 +1,36 @@ +"""Run account registration regression tests against an isolated in-memory database.""" +import importlib +import os +from pathlib import Path +import sys +import unittest + +root = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(root / 'apps')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + +from django.apps import AppConfig +from django.conf import settings + +settings.LOGGING_CONFIG = None +settings.USE_I18N = False +settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}} +settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in ( + 'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common', + 'system_manage', 'models_provider', 'application', 'chat', 'oss', +)] + +import django +django.setup() + +from django.db import connection +from users.models import User, UserActivationApplication, UserRoleAssignment + +with connection.schema_editor() as editor: + for model in (User, UserActivationApplication, UserRoleAssignment): + editor.create_model(model) + +suite = unittest.defaultTestLoader.loadTestsFromName('users.tests.RegistrationTests') +result = unittest.TextTestRunner(verbosity=2).run(suite) +sys.exit(0 if result.wasSuccessful() else 1) diff --git a/scripts/dev_local.py b/scripts/dev_local.py new file mode 100644 index 000000000..c35482c47 --- /dev/null +++ b/scripts/dev_local.py @@ -0,0 +1,40 @@ +"""Single-process development web server; never migrates the shared database.""" +import argparse +import os +from pathlib import Path +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', default='127.0.0.1') + parser.add_argument('--port', type=int, default=8080) + parser.add_argument('--check', action='store_true', help='Check Django configuration without starting the server') + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + os.chdir(root) + sys.path.insert(0, str(root / 'apps')) + os.environ['SERVER_NAME'] = 'web' + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + + # Redirect local logs before Django imports its logging configuration. + from maxkb import const + const.LOG_DIR = str(root / 'data' / 'dev' / 'logs') + from django.conf import settings + if settings.LANGUAGE_CODE.lower() == 'zh-cn': + settings.LANGUAGE_CODE = 'zh-hans' + + import django + django.setup() + from django.core import management + + if args.check: + management.call_command('check') + else: + # No autoreloader: avoid loading a second copy of the Python application. + management.call_command('runserver', f'{args.host}:{args.port}', use_reloader=False, insecure_serving=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/dev_local_model.py b/scripts/dev_local_model.py new file mode 100644 index 000000000..075af90b9 --- /dev/null +++ b/scripts/dev_local_model.py @@ -0,0 +1,35 @@ +"""Single-process weight service with startup preloads; never runs migrations.""" +import argparse +import os +from pathlib import Path +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host') + parser.add_argument('--port', type=int) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + os.chdir(root); sys.path.insert(0, str(root / 'apps')) + os.environ['SERVER_NAME'] = 'local_model' + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + if args.check: + os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0' + from maxkb.const import CONFIG + from django.conf import settings + if settings.LANGUAGE_CODE.lower() == 'zh-cn': + settings.LANGUAGE_CODE = 'zh-hans' + import django + django.setup() + from django.core import management + if args.check: + management.call_command('check') + else: + bind = f"{args.host or CONFIG.get('LOCAL_MODEL_HOST')}:{args.port or CONFIG.get('LOCAL_MODEL_PORT')}" + management.call_command('runserver', bind, use_reloader=False) + + +if __name__ == '__main__': + main() diff --git a/ui/src/api/ai-hub.ts b/ui/src/api/ai-hub.ts new file mode 100644 index 000000000..875dddb4d --- /dev/null +++ b/ui/src/api/ai-hub.ts @@ -0,0 +1,12 @@ +import { get, post, request } from '@/request/index' +import useStore from '@/stores' + +export const hubBase = () => `/workspace/${useStore().user.getWorkspaceId()}/ai_hub` +export const hubGet = (kind: string, id?: string) => get(`${hubBase()}/${kind}${id ? '/' + id : ''}`) +export const hubPost = (kind: string, data: any, id?: string) => post(`${hubBase()}/${kind}${id ? '/' + id : ''}`, data) +export async function hubBinary(kind: 'assets' | 'exports', id?: string, data?: any) { + const response = await request({ url: `${hubBase()}/${kind}${id ? '/' + id + '/content' : ''}`, + method: kind === 'assets' ? 'get' : 'post', data, responseType: 'blob' }) + if (!(response.data instanceof Blob) || response.data.type.includes('json')) throw new Error('图片或导出文件不可用') + return { content: await response.data.arrayBuffer(), mime: response.data.type } +} diff --git a/ui/src/views/knowledge/component/RebuildTaskDrawer.vue b/ui/src/views/knowledge/component/RebuildTaskDrawer.vue new file mode 100644 index 000000000..abd56aa83 --- /dev/null +++ b/ui/src/views/knowledge/component/RebuildTaskDrawer.vue @@ -0,0 +1,182 @@ + + + + + diff --git a/ui/src/views/login/Register.vue b/ui/src/views/login/Register.vue new file mode 100644 index 000000000..7ad5f48cc --- /dev/null +++ b/ui/src/views/login/Register.vue @@ -0,0 +1,158 @@ + + + + + diff --git a/ui/src/views/mindmap/HubManager.vue b/ui/src/views/mindmap/HubManager.vue new file mode 100644 index 000000000..74ba12788 --- /dev/null +++ b/ui/src/views/mindmap/HubManager.vue @@ -0,0 +1,128 @@ + + + + diff --git a/ui/src/views/mindmap/index.vue b/ui/src/views/mindmap/index.vue index 0426743ca..469e85d70 100644 --- a/ui/src/views/mindmap/index.vue +++ b/ui/src/views/mindmap/index.vue @@ -20,13 +20,13 @@ import { ref, onMounted, onBeforeUnmount } from 'vue' import { useRoute } from 'vue-router' import { ElMessage } from 'element-plus' import systemConfigApi from '@/api/system-settings/system-config' -import { hubGet, hubPost } from '@/api/ai-hub' +import { hubGet, hubPost, hubBinary } from '@/api/ai-hub' const mindmapUrl = ref('') const loading = ref(false) const route = useRoute() const frame = ref() -const allowedKinds = new Set(['resources', 'tasks', 'artifacts']) +const allowedKinds = new Set(['resources', 'tasks', 'artifacts', 'assets', 'exports']) function childOrigin() { try { return new URL(mindmapUrl.value, location.href).origin } catch { return '' } } async function openRequestedArtifact() { const id = route.query.artifact @@ -43,6 +43,12 @@ async function handleBridge(event: MessageEvent) { const { id, kind, recordId, method, data, file } = event.data if (typeof id !== 'string' || id.length > 128 || !allowedKinds.has(kind) || !['get', 'post'].includes(method) || (recordId && !/^[a-f0-9-]{36}$/i.test(recordId))) return try { + if (kind === 'assets' || kind === 'exports') { + if ((kind === 'assets' && (method !== 'get' || !recordId)) || (kind === 'exports' && (method !== 'post' || recordId))) throw new Error('请求无效') + const binary = await hubBinary(kind, recordId, data) + frame.value?.contentWindow?.postMessage({ type: 'ai-hub-response', id, data: binary }, event.origin, [binary.content]) + return + } let payload = data if (file) { if (!(file.content instanceof ArrayBuffer) || file.content.byteLength > 20 * 1024 * 1024 || typeof file.name !== 'string') throw new Error('文件无效') diff --git a/ui/src/views/model/component/LocalModelFiles.vue b/ui/src/views/model/component/LocalModelFiles.vue new file mode 100644 index 000000000..f329c679a --- /dev/null +++ b/ui/src/views/model/component/LocalModelFiles.vue @@ -0,0 +1,59 @@ + + + diff --git a/ui/src/views/model/component/ModelLoadingDialog.vue b/ui/src/views/model/component/ModelLoadingDialog.vue new file mode 100644 index 000000000..5fcaa2527 --- /dev/null +++ b/ui/src/views/model/component/ModelLoadingDialog.vue @@ -0,0 +1,89 @@ + + + diff --git a/ui/src/views/system/mindmap/index.vue b/ui/src/views/system/mindmap/index.vue new file mode 100644 index 000000000..52d2b16cb --- /dev/null +++ b/ui/src/views/system/mindmap/index.vue @@ -0,0 +1,18 @@ + + + diff --git a/ui/src/views/system/user-manage/component/ActivationAuditDrawer.vue b/ui/src/views/system/user-manage/component/ActivationAuditDrawer.vue new file mode 100644 index 000000000..ec29046ee --- /dev/null +++ b/ui/src/views/system/user-manage/component/ActivationAuditDrawer.vue @@ -0,0 +1,188 @@ + + + + + diff --git a/ui/src/views/system/user-manage/component/RoleAssignDialog.vue b/ui/src/views/system/user-manage/component/RoleAssignDialog.vue new file mode 100644 index 000000000..4c3ed70e3 --- /dev/null +++ b/ui/src/views/system/user-manage/component/RoleAssignDialog.vue @@ -0,0 +1,161 @@ + + + + + diff --git a/ui/src/views/tool/component/ParserManager.vue b/ui/src/views/tool/component/ParserManager.vue new file mode 100644 index 000000000..1f785b1a6 --- /dev/null +++ b/ui/src/views/tool/component/ParserManager.vue @@ -0,0 +1,69 @@ + + +