完善知识库、模型管理和思维导图功能

v3.2-refact
panyy 2026-10-22 14:48:00 +08:00
parent 6df950ec3f
commit 69eb1eb681
64 changed files with 6126 additions and 3 deletions

View File

@ -0,0 +1,202 @@
"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
import contextlib
import itertools
import queue
import threading
import time
import uuid
from concurrent.futures import Future
def loading_policy(value=None):
value = {} if value is None else value
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
raise ValueError('加载策略无效')
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
'idle_seconds': value.get('idle_seconds', 300)}
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
raise ValueError('加载方式无效')
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
raise ValueError('加载优先级须为0~100的整数')
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
raise ValueError('空闲时间须为30~86400秒')
return result
class ModelRuntime:
def __init__(self, specs, adapter, clock=time.monotonic):
self.specs, self.adapter, self.clock = specs, adapter, clock
self.entries = {}
self.lock = threading.RLock()
self.queue = queue.PriorityQueue()
self.sequence = itertools.count()
self.load_sequence = itertools.count(1)
self.worker = None
def _entry(self, model_id):
model_id = str(model_id)
spec = self.specs(model_id)
spec['policy'] = loading_policy(spec.get('policy'))
entry = self.entries.get(model_id)
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
if entry['leases'] or entry['state'] == 'LOADING':
raise ValueError('模型配置已变更,请等待当前请求结束')
self._discard(entry)
entry = None
if entry is None:
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
'load_order': None}
self.entries[model_id] = entry
previous = entry['spec']['policy']
entry['spec'] = spec
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
entry['generation'] += 1
self._put(entry)
return entry
def _put(self, entry):
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
def start(self):
with self.lock:
if self.worker is None:
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
self.worker.start()
def submit(self, model_id, start=True):
with self.lock:
entry = self._entry(model_id)
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
entry.update(state='UNLOADED', value=None, info={})
if entry['state'] == 'READY':
return entry
if entry['state'] not in ('LOADING', 'QUEUED'):
entry.update(state='QUEUED', error='', future=Future())
entry['generation'] += 1
self._put(entry)
if start:
self.start()
return entry
def preload(self, model_ids):
# Enqueue the complete startup batch before choosing the highest-priority model.
with self.lock:
for model_id in model_ids:
self.submit(model_id, start=False)
self.start()
def _work(self):
while True:
_, order, model_id, generation = self.queue.get()
try:
with self.lock:
entry = self.entries.get(model_id)
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
continue
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
spec = entry['spec']
future = entry['future']
value, info = self.adapter.load(spec)
with self.lock:
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
future.set_result(True)
del value # The loader thread must not keep the last model alive after eviction.
except Exception as exc:
with self.lock:
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
if not entry['future'].done():
entry['future'].set_exception(ValueError(entry['error']))
finally:
self.queue.task_done()
def acquire(self, model_id, remote=False, timeout=600):
with self.lock:
entry = self.submit(model_id)
token = uuid.uuid4().hex
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
future = entry['future'] if entry['state'] != 'READY' else None
try:
if future:
future.result(timeout=timeout)
return token
except Exception:
self.release(model_id, token)
raise
def release(self, model_id, token):
with self.lock:
entry = self.entries.get(str(model_id))
if entry:
entry['leases'].pop(token, None)
entry['last_used'] = self.clock()
@contextlib.contextmanager
def inference(self, model_id):
token = self.acquire(model_id)
try:
with self.lock:
value = self.entries[str(model_id)]['value']
yield value
finally:
self.release(model_id, token)
def _discard(self, entry):
if entry['value'] is not None:
value = entry['value']
self.adapter.unload(entry['spec'], value)
entry['value'] = None
del value
self.adapter.collect()
entry.update(state='UNLOADED', loaded_at=None, info={})
def unload(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
raise ValueError('模型正在使用或加载,请稍后再试')
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
entry['value'] = True
self._discard(entry)
def configure(self, model_id):
with self.lock:
entry = self._entry(model_id)
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
if warm_again:
entry.update(state='UNLOADED', value=None)
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
self.submit(model_id)
def _expire_leases(self, entry):
now = self.clock()
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
def sweep(self):
with self.lock:
for entry in list(self.entries.values()):
self._expire_leases(entry)
policy = entry['spec']['policy']
# Ollama owns its idle timer; do not unload models used by other clients.
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
self.clock() - entry['last_used'] >= policy['idle_seconds']):
self._discard(entry)
def status(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
# Reconcile native residency, including expiration outside this process.
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
info = self.adapter.probe(entry['spec'])
if info:
entry.update(state='READY', info=info, value=True)
elif entry['state'] == 'READY':
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
'active_requests': len(entry['leases']), 'error': entry['error'],
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
'load_order': entry['load_order'], **entry['info']}

View File

@ -0,0 +1,89 @@
"""Authenticated internal model-service RPC; never forwards user-supplied endpoints."""
import contextlib
import asyncio
import hashlib
import hmac
import json
import os
import time
import uuid
from functools import lru_cache
import requests
@lru_cache(maxsize=1)
def _secret():
if os.environ.get('SERVER_NAME') == 'local_model':
from local_model.models.system_setting import SystemSetting
else:
from system_manage.models import SystemSetting
meta = SystemSetting.objects.filter(type=1).values_list('meta', flat=True).first()
if not meta or not meta.get('value'):
raise ValueError('模型服务认证未配置')
return hashlib.sha256(('model-runtime-v1:' + meta['value']).encode()).digest()
def signed_request(payload):
data = {**payload, 'timestamp': time.time(), 'nonce': uuid.uuid4().hex}
body = json.dumps(data, separators=(',', ':'), sort_keys=True).encode()
signature = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
return body, signature
def verify_request(body, signature):
expected = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
if not signature or not hmac.compare_digest(expected, signature):
raise ValueError('模型服务认证失败')
data = json.loads(body)
if abs(time.time() - float(data.get('timestamp', 0))) > 60:
raise ValueError('模型服务请求已过期')
return data
def runtime_request(operation, model_id, **extra):
from maxkb.const import CONFIG
body, signature = signed_request({'operation': operation, 'model_id': str(model_id), **extra})
host = CONFIG.get('LOCAL_MODEL_HOST')
url = f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{host}:{CONFIG.get("LOCAL_MODEL_PORT")}{CONFIG.get_admin_path()}/api/model_runtime'
try:
response = requests.post(url, data=body, headers={'Content-Type': 'application/json',
'X-Model-Runtime-Signature': signature},
timeout=(5, 620 if operation == 'acquire' else 15))
result = response.json()
except (requests.RequestException, ValueError) as exc:
raise ValueError('本地模型服务不可用') from exc
if response.status_code != 200 or result.get('code') != 200:
raise ValueError(result.get('message') or '模型服务请求失败')
return result['data']
@contextlib.contextmanager
def inference_lease(model_id):
if not model_id:
# Credential validation has no saved resource and must keep its original behavior.
yield None
return
data = runtime_request('acquire', model_id)
try:
yield data
finally:
try:
runtime_request('release', model_id, lease=data['lease'])
except ValueError:
pass # Server expires abandoned leases; never hide the inference result.
@contextlib.asynccontextmanager
async def async_inference_lease(model_id):
if not model_id:
yield None
return
data = await asyncio.to_thread(runtime_request, 'acquire', model_id)
try:
yield data
finally:
try:
await asyncio.to_thread(runtime_request, 'release', model_id, lease=data['lease'])
except ValueError:
pass

View File

@ -0,0 +1,27 @@
"""按完整表格行分段;表头重复,重叠行不跨表/工作表。"""
def split_table_rows(header, rows, limit, overlap=0):
limit, overlap = int(limit), int(overlap)
if limit <= 0 or not 0 <= overlap < limit:
raise ValueError("Invalid table splitting limits")
chunks, current = [], []
length = len(header)
for row in rows:
if current and length + len(row) > limit:
chunks.append({"content": header + "".join(current), "title": ""})
carry, carried = [], 0
if overlap:
for previous in reversed(current):
carry.insert(0, previous)
carried += len(previous)
if carried >= overlap:
break
# 必须留出至少一条新行的空间,不能只重复旧行。
while carry and len(header) + sum(map(len, carry)) + len(row) > limit:
carry.pop(0)
current = carry
length = len(header) + sum(map(len, current))
current.append(row)
length += len(row)
if current:
chunks.append({"content": header + "".join(current), "title": ""})
return chunks

View File

@ -0,0 +1,65 @@
"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
import math
import unicodedata
from langchain_core.embeddings import Embeddings
def validate_params(params):
if (not isinstance(params, dict) or len(params) > 52 or
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
raise ValueError('可选参数最多50项')
result, names = {}, set()
for key, value in params.items():
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
raise ValueError('参数名称须为1~64个字符')
key = key.strip()
normalized = unicodedata.normalize('NFKC', key).casefold()
if normalized in names:
raise ValueError('参数名称不能重复')
names.add(normalized)
if value is not None and (not isinstance(value, (str, int, float, bool)) or
isinstance(value, str) and len(value) > 2000 or
isinstance(value, float) and not math.isfinite(value)):
raise ValueError('参数值须为2000字以内的文本或数值')
result[key] = value
batch = result.get('batch_size')
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
raise ValueError('批处理数量须为1~256的整数')
if result.get('normalize') is not None and type(result['normalize']) is not bool:
raise ValueError('向量归一化参数无效')
for key in ('query_prefix', 'document_prefix'):
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
raise ValueError('向量指令须为2000字以内的文本')
return {key: value for key, value in result.items() if value is not None and
(value != '' or key not in {'query_prefix', 'document_prefix'})}
def knowledge_params(knowledge):
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
return {key: value for key, value in params.items()
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
class ConfiguredEmbedding(Embeddings):
def __init__(self, model, params):
self.model = model
self.params = params
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
def _vector(self, vector):
if not self.params.get('normalize'):
return vector
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
return [float(v) / norm for v in vector] if norm else vector
def embed_query(self, text):
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
def embed_documents(self, texts):
prefix = self.params.get('document_prefix', '')
result = []
for start in range(0, len(texts), self.batch_size):
result.extend(self._vector(v) for v in self.model.embed_documents(
[prefix + text for text in texts[start:start + self.batch_size]]))
return result

View File

@ -0,0 +1,39 @@
# Generated for independent knowledge index rebuild task
import uuid_utils.compat
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0003_tag_documenttag'),
('users', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='KnowledgeRebuildTask',
fields=[
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')),
('status', models.CharField(choices=[('PENDING', '待执行'), ('RUNNING', '执行中'), ('SUCCESS', '成功'), ('PARTIAL', '部分成功'), ('FAILED', '失败')], db_index=True, default='PENDING', max_length=20, verbose_name='任务状态')),
('config_snapshot', models.JSONField(default=dict, verbose_name='配置快照')),
('document_total', models.IntegerField(default=0, verbose_name='文档总数')),
('document_success', models.IntegerField(default=0, verbose_name='成功文档数')),
('document_failed', models.IntegerField(default=0, verbose_name='失败文档数')),
('error_message', models.TextField(blank=True, default='', verbose_name='失败原因')),
('started_at', models.DateTimeField(blank=True, null=True, verbose_name='开始时间')),
('finished_at', models.DateTimeField(blank=True, null=True, verbose_name='结束时间')),
('celery_task_id', models.CharField(blank=True, default='', max_length=128, verbose_name='Celery任务id')),
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='rebuild_tasks', to='knowledge.knowledge', verbose_name='知识库')),
('operator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='操作人')),
],
options={
'db_table': 'knowledge_rebuild_task',
'indexes': [models.Index(fields=['knowledge', 'status'], name='knowledge_r_knowled_0a6e2f_idx')],
},
),
]

View File

@ -0,0 +1,17 @@
# Generated for rebuild task attempt polling
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0004_knowledge_rebuild_task'),
]
operations = [
migrations.AddField(
model_name='knowledgerebuildtask',
name='attempt',
field=models.IntegerField(default=0, verbose_name='汇总轮询次数'),
),
]

View File

@ -0,0 +1,215 @@
# coding=utf-8
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
from collections.abc import Mapping
import re
from django.db import transaction
from django.db.models import QuerySet
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import Knowledge
METADATA_FIELDS = [
{"field": "business_domain", "label": "业务领域"},
{"field": "data_source", "label": "数据来源"},
{"field": "owner_dept", "label": "责任部门"},
{"field": "business_version", "label": "业务版本"},
]
METADATA_MAX_LEN = 64
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
query = QuerySet(Knowledge)
if lock:
query = query.select_for_update()
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
return knowledge
def _creator_candidates(workspace_id):
from users.models import User, UserActivationApplication
from common.database_model_manage.database_model_manage import DatabaseModelManage
from django.db.models import Q
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
query = User.objects.filter(is_active=True).exclude(id__in=pending)
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
if membership is not None:
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
return query
def _metadata_fields(business):
# An explicit empty list means all custom fields were removed; do not revive old fields.
if isinstance(business.get('custom_fields'), list):
return business['custom_fields']
return [{'name': item['label'], 'value': business[item['field']]}
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
def _validate_custom_fields(items):
import unicodedata
if not isinstance(items, list) or len(items) > 50:
raise AppApiException(500, '元数据最多50项')
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
result, names = [], set()
for item in items:
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
raise AppApiException(500, '元数据需包含名称和值')
name, value = item['name'], item['value']
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
raise AppApiException(500, '元数据名称须为1~64个字符')
name = name.strip()
normalized = unicodedata.normalize('NFKC', name).casefold()
if normalized in reserved:
raise AppApiException(500, '不能使用固定属性名称')
if normalized in names:
raise AppApiException(500, '元数据名称不能重复')
if not isinstance(value, str) or len(value) > 2000:
raise AppApiException(500, '元数据值须为2000字以内的文本')
names.add(normalized)
result.append({'name': name, 'value': value.strip()})
return result
class KnowledgeMetadataSerializer(serializers.Serializer):
"""Fixed system attributes plus freely named business metadata; save atomically."""
@staticmethod
def one(workspace_id: str, knowledge_id: str):
from users.models import User
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = (knowledge.meta or {}).get('business_metadata') or {}
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
for u in _creator_candidates(workspace_id).order_by('nick_name')]
if creator and not any(u['id'] == str(creator.id) for u in candidates):
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
return {
'knowledge_id': str(knowledge.id),
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
'creator_name': (creator.nick_name or creator.username) if creator else '',
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
'creator_options': candidates,
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
}
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, '元数据必须为属性对象')
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
if set(data) - (set(legacy) | {'fields', 'user_id'}):
raise AppApiException(500, '包含不支持的元数据属性')
if 'fields' in data and set(data) & set(legacy):
raise AppApiException(500, '不能混用新旧元数据格式')
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
values = {}
for key, label in legacy.items():
if key not in data:
continue
value = '' if data[key] is None else data[key]
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
raise AppApiException(500, f'{label}须为64字以内的文本')
values[key] = value.strip()
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
try:
creator_id = serializers.UUIDField().run_validation(data['user_id'])
except serializers.ValidationError:
raise AppApiException(500, '请选择有效创建者')
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
raise AppApiException(500, '创建者须为已激活且启用的成员')
knowledge.user_id = creator_id
meta = dict(knowledge.meta or {})
business = dict(meta.get('business_metadata') or {})
if fields is not None:
business['custom_fields'] = fields
elif 'custom_fields' in business:
current = list(_metadata_fields(business))
for key, value in values.items():
existing = next((item for item in current if item['name'] == legacy[key]), None)
if existing is not None:
existing['value'] = value
elif value:
current.append({'name': legacy[key], 'value': value})
business['custom_fields'] = _validate_custom_fields(current)
business.update(values)
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
updated_at=_fmt(timezone.now()))
meta['business_metadata'] = business
knowledge.meta = meta
knowledge.save(update_fields=['meta', 'user', 'update_time'])
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
class KnowledgeSegmentationSerializer(serializers.Serializer):
"""知识库分段规则:查看 / 修改。
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
保存后新建文档上传时默认使用该规则,上传时可覆盖。
"""
DEFAULTS = {"chunk_size": 4096, "chunk_overlap": 0}
@staticmethod
def one(workspace_id: str, knowledge_id: str):
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = dict((knowledge.meta or {}).get('segmentation') or {})
result = {
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
}
result.update({
"updated_by": data.get("updated_by_name") or "",
"updated_at": data.get("updated_at") or "",
"knowledge_id": str(knowledge.id),
})
return result
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, "分段规则必须为属性对象")
def integer(value):
if isinstance(value, bool) or not isinstance(value, (int, str)):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
return int(value)
chunk_size = integer(data.get("chunk_size"))
chunk_overlap = integer(data.get("chunk_overlap", 0))
if chunk_size < 50 or chunk_size > 100000:
raise AppApiException(500, "分段长度取值范围为 50~100000")
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
meta = dict(knowledge.meta or {})
meta['segmentation'] = {
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap,
"updated_by": str(user.id),
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
"updated_at": _fmt(timezone.now()),
}
knowledge.meta = meta
knowledge.save(update_fields=["meta", "update_time"])
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)

View File

@ -0,0 +1,129 @@
# coding=utf-8
"""知识库独立索引重建任务序列化器。"""
from django.db import transaction
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
from knowledge.task.rebuild import rebuild_knowledge_index
from models_provider.models import Model
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def serialize_task(task: KnowledgeRebuildTask):
total = task.document_total or 0
success = task.document_success or 0
failed = task.document_failed or 0
processed = success + failed
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
return {
"id": str(task.id),
"knowledge_id": str(task.knowledge_id),
"workspace_id": task.workspace_id,
"status": task.status,
"status_label": status_label,
"config_snapshot": task.config_snapshot or {},
"document_total": total,
"document_success": success,
"document_failed": failed,
"progress": progress,
"error_message": task.error_message or "",
"created_at": _fmt(task.create_time),
"started_at": _fmt(task.started_at),
"finished_at": _fmt(task.finished_at),
}
class KnowledgeRebuildSerializer(serializers.Serializer):
@staticmethod
def create(workspace_id: str, knowledge_id: str, user_id: str):
# 锁住知识库行,串行化同一知识库的检查和任务创建。
with transaction.atomic():
knowledge = Knowledge.objects.select_for_update().filter(
id=knowledge_id, workspace_id=workspace_id
).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
if KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id,
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).exists():
raise AppApiException(500, "该知识库已有重建任务在执行")
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
if model is None:
raise AppApiException(500, "知识库未配置可用的向量模型")
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge_id).values_list("id", flat=True)]
if not document_ids:
raise AppApiException(500, "知识库没有可重建的文档")
snapshot = {
"embedding_model_id": str(model.id),
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
"embedding_model_name": model.name,
"embedding_model": model.model_name,
"knowledge_name": knowledge.name,
"document_ids": document_ids,
"document_total": len(document_ids),
"snapshot_at": _fmt(timezone.now()),
}
task = KnowledgeRebuildTask.objects.create(
knowledge_id=knowledge_id, workspace_id=workspace_id,
status=RebuildStatus.PENDING, config_snapshot=snapshot,
document_total=len(document_ids), operator_id=user_id,
)
def dispatch():
try:
result = rebuild_knowledge_index.delay(str(task.id))
# 不覆盖 worker 已经更新的状态。
KnowledgeRebuildTask.objects.filter(id=task.id).update(
celery_task_id=getattr(result, "id", "") or "")
except Exception as exc:
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
finished_at=timezone.now(), update_time=timezone.now())
transaction.on_commit(dispatch)
task.refresh_from_db()
return serialize_task(task)
@staticmethod
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
query_set = KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id, workspace_id=workspace_id
).order_by("-create_time")
total = query_set.count()
start = max(0, (current_page - 1) * page_size)
records = [serialize_task(item) for item in query_set[start:start + page_size]]
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
@staticmethod
def one(workspace_id: str, knowledge_id: str, task_id: str):
task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if task is None:
raise AppApiException(500, "重建任务不存在")
return serialize_task(task)
@staticmethod
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
old_task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if old_task is None:
raise AppApiException(500, "重建任务不存在")
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)

View File

@ -0,0 +1,146 @@
# coding=utf-8
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
from django.db import transaction
from django.utils import timezone
from common.db.sql_execute import sql_execute
from common.event.listener_manage import ListenerManagement
from common.utils.logger import maxkb_logger
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
TaskType, State, Status)
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
from ops import celery_app
POLL_SECONDS = 5
MAX_POLLS = 120
def _index_exists(knowledge_id: str) -> bool:
rows = sql_execute(
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
[f"embedding_hnsw_idx_{knowledge_id}"]
)
return bool(rows)
def _fail(task_id, message):
KnowledgeRebuildTask.objects.filter(
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).update(status=RebuildStatus.FAILED, error_message=message,
finished_at=timezone.now(), update_time=timezone.now())
@celery_app.task(name="celery:rebuild_knowledge_index")
def rebuild_knowledge_index(task_id: str):
# 重复投递不能重复清空向量或重复派发。
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
attempt=0, update_time=timezone.now())
if not claimed:
return
try:
task = KnowledgeRebuildTask.objects.get(id=task_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if knowledge is None:
raise ValueError("知识库不存在")
snapshot = task.config_snapshot or {}
model_id = snapshot.get("embedding_model_id")
if not model_id or str(knowledge.embedding_model_id) != model_id:
raise ValueError("向量模型已变更,请重新发起任务")
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
raise ValueError('嵌入参数已变更,请重新发起任务')
# 兼容历史任务;新任务在受理时即固定文档范围。
document_ids = snapshot.get("document_ids")
if document_ids is None:
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge.id).values_list("id", flat=True)]
snapshot["document_ids"] = document_ids
task.config_snapshot = snapshot
task.save(update_fields=["config_snapshot", "update_time"])
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
if not documents or len(documents) != len(document_ids):
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
raise ValueError("文档正在向量化,请完成后重新发起")
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
drop_knowledge_index(knowledge_id=str(knowledge.id))
task.document_total = len(document_ids)
task.save(update_fields=["document_total", "update_time"])
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
ListenerManagement.update_status(
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
TaskType.EMBEDDING, State.PENDING)
from knowledge.task.embedding import embedding_by_document
dispatch_errors = []
for document in documents:
try:
embedding_by_document.delay(str(document.id), model_id)
except Exception as exc:
ListenerManagement.update_status(
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
dispatch_errors.append(f"{document.name}:{exc}")
if dispatch_errors:
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
task.save(update_fields=["error_message", "update_time"])
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
except Exception as exc:
_fail(task_id, f"重建任务启动失败:{exc}")
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
@celery_app.task(name="celery:finalize_rebuild_task")
def finalize_rebuild_task(task_id: str):
try:
with transaction.atomic():
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
if task is None or task.status != RebuildStatus.RUNNING:
return
snapshot = task.config_snapshot or {}
document_ids = snapshot.get("document_ids", [])
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
success = states.count(State.SUCCESS)
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
for state in states) + len(document_ids) - len(documents)
pending = len(document_ids) - success - failed
task.document_total = len(document_ids)
task.document_success = success
task.document_failed = failed
if pending:
task.attempt += 1
if task.attempt < MAX_POLLS:
task.save(update_fields=["document_total", "document_success", "document_failed",
"attempt", "update_time"])
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
(task_id,), countdown=POLL_SECONDS))
return
task.status = RebuildStatus.FAILED
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
else:
knowledge_id = str(task.knowledge_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
raise ValueError("向量模型已变更,请重新发起任务")
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
create_knowledge_index(knowledge_id=knowledge_id)
index_ready = _index_exists(knowledge_id)
if success and failed == 0 and index_ready:
task.status = RebuildStatus.SUCCESS
task.error_message = ""
elif success and failed and index_ready:
task.status = RebuildStatus.PARTIAL
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
else:
task.status = RebuildStatus.FAILED
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
task.finished_at = timezone.now()
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
"error_message", "finished_at", "attempt", "update_time"])
except Exception as exc:
_fail(task_id, f"重建结果汇总失败:{exc}")
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)

View File

@ -0,0 +1,3 @@
"""Celery 任务发现入口,确保独立重建和向量化任务被 Worker 注册。"""
from knowledge.task import embedding # noqa: F401
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task # noqa: F401

View File

@ -0,0 +1,73 @@
import unittest
from types import SimpleNamespace
from knowledge.embedding_params import ConfiguredEmbedding, validate_params, knowledge_params
class RecordingModel:
batch_size = 10
def __init__(self):
self.calls = []
def embed_query(self, text):
self.calls.append(('query', text))
return [3., 4.]
def embed_documents(self, texts):
self.calls.append(('documents', texts))
return [[3., 4.] for _ in texts]
class EmbeddingParamsTests(unittest.TestCase):
def test_inference_uses_separate_instructions_and_same_normalization(self):
base = RecordingModel()
model = ConfiguredEmbedding(base, {'normalize': True, 'query_prefix': '检索:', 'document_prefix': '正文:'})
self.assertEqual(model.embed_query('内容'), [.6, .8])
self.assertEqual(model.embed_documents(['内容']), [[.6, .8]])
self.assertEqual(base.calls, [('query', '检索:内容'), ('documents', ['正文:内容'])])
def test_batch_size_is_applied_to_real_calls(self):
base = RecordingModel()
model = ConfiguredEmbedding(base, {'batch_size': 2})
self.assertEqual(len(model.embed_documents(['a', 'b', 'c', 'd', 'e'])), 5)
self.assertEqual([len(call[1]) for call in base.calls], [2, 2, 1])
def test_knowledge_overrides_do_not_leak_between_shared_models(self):
base = RecordingModel()
first = ConfiguredEmbedding(base, {'query_prefix': 'A'})
second = ConfiguredEmbedding(base, {'query_prefix': 'B'})
first.embed_query('x'); second.embed_query('x'); first.embed_query('x')
self.assertEqual(base.calls, [('query', 'Ax'), ('query', 'Bx'), ('query', 'Ax')])
self.assertEqual(base.batch_size, 10)
def test_invalid_inputs_are_rejected(self):
for params in ([], {'batch_size': True}, {'batch_size': 0}, {'batch_size': 257},
{'batch_size': 2.5}, {'normalize': 'true'}, {'query_prefix': 1},
{'document_prefix': 'x' * 2001}, {'': 'value'}, {'custom': {}},
{'custom': ['value']}, {'x': 1, 'X': 2}, {'custom': float('nan')},
{'a' * 65: 'value'}, {str(i): 'value' for i in range(51)}):
with self.subTest(params=params), self.assertRaises(ValueError):
validate_params(params)
def test_custom_names_are_saved_but_unsupported_options_do_not_reach_inference(self):
params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'}
self.assertEqual(validate_params(params), params)
active = knowledge_params(SimpleNamespace(meta={'embedding_params': params}))
self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:'})
base = RecordingModel()
model = ConfiguredEmbedding(base, active)
model.embed_query('问题')
self.assertEqual(base.calls, [('query', '检索:问题')])
def test_legacy_and_empty_params_keep_original_behavior(self):
self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {})
self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {})
base = RecordingModel()
self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [3., 4.])
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), [])
def test_zero_vector_is_safe(self):
base = RecordingModel()
base.embed_query = lambda _: [0., 0.]
self.assertEqual(ConfiguredEmbedding(base, {'normalize': True}).embed_query('x'), [0., 0.])

View File

@ -0,0 +1,288 @@
"""Real isolated persistence and document parsing regression checks."""
import io
import importlib
import unittest
from types import SimpleNamespace
from unittest.mock import patch, Mock
from django.test import TestCase
from django.core.files.uploadedfile import SimpleUploadedFile
from common.exception.app_exception import AppApiException
from common.utils.split_model import smart_split_paragraph, SplitModel
from common.utils.table_split import split_table_rows
from knowledge.models import KnowledgeFolder, Knowledge
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
class MetadataTests(TestCase):
def setUp(self):
KnowledgeFolder.objects.create(id='default', name='root')
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
meta={'internal': {'preserve': True}})
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
def test_metadata_persists_and_preserves_other_settings(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
self.assertEqual(self.k.meta['internal'], {'preserve': True})
result = Metadata.one('default', str(self.k.id))
self.assertEqual(len(result['fields']), 1)
self.assertEqual(result['updated_by'], '测试员')
self.assertTrue(result['updated_at'])
def test_partial_metadata_update_does_not_clear_other_fields(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
old = Metadata.one('default', str(self.k.id))
self.assertEqual(old['fields'][0]['name'], '业务领域')
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
self.k.refresh_from_db()
self.assertEqual(self.k.meta['internal'], {'preserve': True})
def test_custom_field_validation_does_not_write(self):
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
[{'name': str(i), 'value': ''} for i in range(51)]]
for fields in cases:
with self.subTest(fields=fields), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
with self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
from users.models import User
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
original_time = self.k.create_time
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.k.refresh_from_db()
self.assertEqual(self.k.user_id, creator.id)
self.assertEqual(self.k.create_time, original_time)
self.assertEqual(result['creator_name'], '新创建者')
self.assertEqual(result['user_id'], str(creator.id))
self.assertTrue(result['update_time'])
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
from users.models import User, UserActivationApplication
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
for status in ('PENDING', 'REJECTED'):
application.status = status; application.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
application.status = 'APPROVED'; application.save()
creator.is_active = False; creator.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
self.k.refresh_from_db()
self.assertIsNone(self.k.user_id)
self.assertNotIn('business_metadata', self.k.meta)
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
from knowledge.serializers.knowledge import KnowledgeSerializer
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
with patch.object(operation, 'is_valid'):
operation.edit({'embedding_params': params}, select_one=False)
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
self.k.refresh_from_db(); other.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
self.assertNotIn('embedding_params', other.meta)
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
def test_general_settings_cannot_restore_stale_metadata(self):
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
self.k.refresh_from_db()
protected = self.k.meta['business_metadata']
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata'], protected)
self.assertEqual(self.k.meta['selector'], 'body')
self.assertNotIn('parser_id', self.k.meta)
def test_invalid_types_and_unknown_fields_do_not_write(self):
for data in ([1], {'business_domain': 123}, {'business_domain': False},
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
with self.subTest(data=data), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, data)
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_workspace_boundary(self):
for fn in (Metadata.one, Rules.one):
with self.assertRaises(AppApiException):
fn('other', str(self.k.id))
def test_segmentation_and_metadata_updates_preserve_each_other(self):
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
def test_segmentation_rejects_invalid_and_fractional_values(self):
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
(100,1.2), (True,0), (100,False), ('100.5',0)):
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
from knowledge.serializers.document import DocumentSerializers
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
f=SimpleUploadedFile('test.txt',b'test')
serializer.parse({'file':[f]})
self.assertEqual(split.call_args.args[-2:], (100,20))
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
self.assertEqual(split.call_args.args[-2:], (200,0))
with self.assertRaises(AppApiException):
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
class ParserTests(unittest.TestCase):
text=''.join(chr(0x4e00+i) for i in range(500))
def check_overlap(self, parts):
self.assertGreater(len(parts),1)
self.assertTrue(all(len(p)<=100 for p in parts))
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
def test_plain_text_overlap(self):
self.check_overlap(smart_split_paragraph(self.text,100,20))
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
result=PdfSplitHandle.handle_toc(doc,100,20)
self.check_overlap([p['content'] for p in result])
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
import ast,inspect,textwrap
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
self.assertTrue(calls)
self.assertTrue(all(len(n.args)==3 for n in calls))
def test_table_overlap_preserves_rows_headers_and_termination(self):
header='| id | value |\n| --- | --- |\n'
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
chunks=split_table_rows(header,iter(rows),120,20)
self.assertGreater(len(chunks),1)
self.assertLess(len(chunks),len(rows))
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
def test_oversized_table_row_is_not_truncated(self):
row='| '+'x'*500+' |\n'
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
self.assertTrue(any(row in p['content'] for p in result))
self.assertIn('last',result[-1]['content'])
def test_csv_and_xlsx_use_row_overlap(self):
import openpyxl
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
book=openpyxl.Workbook();sheet=book.active
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
out=io.BytesIO();book.save(out)
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
f=SimpleUploadedFile(name,data)
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
chunks=result[0]['content'] if isinstance(result,list) else result['content']
self.assertGreater(len(chunks),1)
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_xls_uses_row_overlap(self):
from common.handle.impl.text.xls_split_handle import handle_sheet
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
chunks=handle_sheet('test',sheet,120,20)['content']
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
manager=Mock();manager.using.return_value.filter.return_value=[model]
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
model.save.assert_called_once()
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
import json
import httpx
from openai import OpenAI
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
from langchain_core.messages import HumanMessage
requests=[]
def transport(request):
body=json.loads(request.content);requests.append(body)
if body.get('stream'):
events=[]
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
events.append('data: '+json.dumps(item)+'\n\n')
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
text=''.join(events)+'data: [DONE]\n\n')
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
model=BaiLianChatModel.new_instance('LLM','test',{
'api_base':'http://example.invalid/v1','api_key':'test-only'},
temperature=0.23,top_p=0.42,max_tokens=10)
model.client=client.chat.completions
model.root_client=client
model.streaming=False
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
model.streaming=True
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
self.assertEqual(len(requests),2)
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
for req in requests:
self.assertEqual(req['temperature'],0.23)
self.assertEqual(req['top_p'],0.42)
def test_new_knowledge_tasks_use_isolated_queue(self):
from ops import celery_app
import knowledge.tasks
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
self.assertIn(name,celery_app.tasks)
route=celery_app.amqp.router.route({},name)
self.assertEqual(route['queue'].name,'uniskb_knowledge')

View File

@ -0,0 +1,187 @@
"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls."""
import unittest
import uuid
from types import SimpleNamespace
from unittest.mock import patch, Mock
from contextlib import ExitStack
from django.db import transaction, connection
from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS
from common.exception.app_exception import AppApiException
class RebuildTests(unittest.TestCase):
def setUp(self):
with connection.cursor() as cursor:
cursor.execute('DELETE FROM knowledge_rebuild_task')
self.kid = str(uuid.uuid4())
self.docid = str(uuid.uuid4())
self.modelid = str(uuid.uuid4())
self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test')
self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2')
self.stack = ExitStack()
self.addCleanup(self.stack.close)
def mock(name):
return self.stack.enter_context(patch(name))
self.k = mock('knowledge.serializers.rebuild.Knowledge.objects')
self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge
self.k.filter.return_value.first.return_value = self.knowledge
self.docs = mock('knowledge.serializers.rebuild.Document.objects')
self.docs.filter.return_value.values_list.return_value = [self.docid]
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document])
self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects')
self.models = mock('knowledge.serializers.rebuild.Model.objects')
self.models.filter.return_value.first.return_value = SimpleNamespace(
id=self.modelid, name='embedding', model_name='test-model')
self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay')
self.delay.return_value.id = 'queued-test'
self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async')
self.listener = mock('knowledge.task.rebuild.ListenerManagement')
self.drop = mock('knowledge.task.rebuild.drop_knowledge_index')
self.create_index = mock('knowledge.task.rebuild.create_knowledge_index')
self.index_exists = mock('knowledge.task.rebuild._index_exists')
self.index_exists.return_value = True
self.child = mock('knowledge.task.embedding.embedding_by_document.delay')
def create(self):
return KnowledgeRebuildSerializer.create('default', self.kid, None)
def task(self, status=RebuildStatus.RUNNING):
return KnowledgeRebuildTask.objects.create(
knowledge_id=self.kid, workspace_id='default', status=status, document_total=1,
config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]})
def test_dispatch_only_after_commit(self):
with transaction.atomic():
result = self.create()
self.delay.assert_not_called()
self.assertEqual(result['status'], RebuildStatus.PENDING)
self.delay.assert_called_once()
self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test')
def test_rollback_does_not_dispatch(self):
with self.assertRaises(ValueError):
with transaction.atomic():
self.create()
raise ValueError('rollback')
self.delay.assert_not_called()
self.assertFalse(KnowledgeRebuildTask.objects.exists())
def test_broker_failure_persists_failed_record(self):
self.delay.side_effect = RuntimeError('broker offline')
result = self.create()
self.assertEqual(result['status'], RebuildStatus.FAILED)
self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message)
def test_active_task_rejected_under_knowledge_lock(self):
self.task()
with self.assertRaises(AppApiException):
self.create()
self.k.select_for_update.assert_called_once()
self.assertEqual(KnowledgeRebuildTask.objects.count(), 1)
def test_empty_knowledge_rejected(self):
self.docs.filter.return_value.values_list.return_value = []
with self.assertRaises(AppApiException):
self.create()
self.assertFalse(KnowledgeRebuildTask.objects.exists())
def test_success_cannot_be_retried(self):
task = self.task(RebuildStatus.SUCCESS)
with self.assertRaises(AppApiException):
KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None)
def test_failed_retry_creates_new_record(self):
old = self.task(RebuildStatus.FAILED)
result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None)
self.assertNotEqual(result['id'], str(old.id))
self.assertEqual(KnowledgeRebuildTask.objects.count(), 2)
def test_duplicate_worker_delivery_does_not_delete_vectors(self):
task = self.task()
rebuild_knowledge_index.run(str(task.id))
self.listener.delete_embedding_by_knowledge.assert_not_called()
def test_worker_dispatches_independent_monitor(self):
task = self.task(RebuildStatus.PENDING)
rebuild_knowledge_index.run(str(task.id))
self.child.assert_called_once_with(self.docid, self.modelid)
self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid])
self.poll.assert_called_once()
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.RUNNING)
def test_document_dispatch_failure_is_counted(self):
task = self.task(RebuildStatus.PENDING)
self.child.side_effect = RuntimeError('cannot queue')
rebuild_knowledge_index.run(str(task.id))
self.listener.update_status.assert_any_call(
self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE)
self.document.status = 'nn3'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.assertEqual(task.document_failed, 1)
self.assertIn('cannot queue', task.error_message)
def test_success_requires_index(self):
task = self.task()
self.index_exists.return_value = False
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_successful_finalization(self):
task = self.task()
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.SUCCESS)
self.assertEqual(task.document_success, 1)
self.assertIsNotNone(task.finished_at)
def test_pending_persists_progress_and_schedules_next_poll(self):
task = self.task()
other = SimpleNamespace(id=uuid.uuid4(), status='nn0')
task.config_snapshot['document_ids'].append(str(other.id))
task.save()
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other])
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.RUNNING)
self.assertEqual(task.document_success, 1)
self.poll.assert_called_once()
def test_timeout_records_failure(self):
task = self.task()
task.attempt = MAX_POLLS - 1
task.save()
self.document.status = 'nn0'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.assertIn('超时', task.error_message)
def test_ignored_document_is_failure(self):
task = self.task()
self.document.status = 'nnn'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.document_failed, 1)
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_deleted_document_is_not_silent_success(self):
task = self.task()
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([])
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.document_failed, 1)
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_model_change_fails_before_destructive_work(self):
task = self.task(RebuildStatus.PENDING)
self.knowledge.embedding_model_id = str(uuid.uuid4())
rebuild_knowledge_index.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.listener.delete_embedding_by_knowledge.assert_not_called()

View File

@ -0,0 +1,172 @@
"""Weight owner for local resources; shared by both assistants through model IDs."""
import gc
import hashlib
import json
import logging
import threading
import time
from urllib.parse import urlsplit, urlunsplit
import requests
from django.db import close_old_connections
from common.model_runtime import ModelRuntime, loading_policy
logger = logging.getLogger('maxkb')
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
def driver_for(provider, model_type):
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
def model_spec(model_id):
from local_model.models import Model
from local_model.serializers.rsa_util import rsa_long_decrypt
try:
model = Model.objects.get(id=model_id, status='SUCCESS')
except Model.DoesNotExist:
raise ValueError('模型不存在或尚不可用')
driver = driver_for(model.provider, model.model_type)
if not driver:
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
class WeightAdapter:
@staticmethod
def _request(spec, route, data=None):
url = urlsplit(spec['credential'].get('api_base', ''))
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
raise ValueError('模型服务地址无效')
path = url.path.rstrip('/')
if path.endswith('/v1'):
path = path[:-3]
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
key = spec['credential'].get('api_key')
headers = {'Authorization': 'Bearer ' + key} if key else {}
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
response.raise_for_status()
return response.json()
def probe(self, spec):
try:
tags = self._request(spec, '/api/ps').get('models', [])
except requests.RequestException as exc:
raise ValueError('模型推理服务不可用') from exc
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
if item:
vram = item.get('size_vram', 0)
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
def load(self, spec):
close_old_connections()
try:
if spec['driver'] == 'ollama':
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
'keep_alive': keep_alive})
info = self.probe(spec)
if not info:
raise ValueError('模型未驻留')
logger.info('Model weights ready: %s', spec['id'])
return True, info
from local_model.serializers.model_apply_serializers import get_local_model
instance = get_local_model(spec['model'], use_local=True)
if spec['type'] == 'EMBEDDING':
instance.embed_query('模型加载检查')
else:
from langchain_core.documents import Document
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
info = {'device': device}
try:
import torch
if device.startswith('cuda'):
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
except ImportError:
pass
logger.info('Model weights ready: %s', spec['id'])
return instance, info
finally:
close_old_connections()
def unload(self, spec, value):
if spec['driver'] == 'ollama':
# Refuse to unload a native model shared by another configured resource.
from local_model.models import Model
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
for _ in range(50):
if self.probe(spec) is None:
return
time.sleep(.2)
raise ValueError('模型卸载未完成,请稍后重试')
@staticmethod
def collect():
gc.collect()
try:
import torch
if torch.cuda.is_initialized():
torch.cuda.empty_cache()
except ImportError:
pass
@staticmethod
def error(exc):
name = type(exc).__name__
logger.warning('Model weight load failed: %s', name)
if 'OutOfMemory' in name:
return '模型加载失败:显存或内存不足'
return '模型加载失败,请检查模型文件、运行设备及服务状态'
manager = ModelRuntime(model_spec, WeightAdapter())
_startup_lock = threading.Lock()
_started = False
def start_runtime():
global _started
with _startup_lock:
if _started:
return
_started = True
def run():
from django.apps import apps
while not apps.ready:
time.sleep(.05)
from local_model.models import Model
try:
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
manager.preload(ids)
logger.info('Startup model preloads queued: %s', len(ids))
except Exception as exc:
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
finally:
close_old_connections()
while True:
time.sleep(10)
try:
manager.sweep()
except Exception as exc:
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
finally:
close_old_connections()
threading.Thread(target=run, name='model-startup', daemon=True).start()

View File

@ -0,0 +1,48 @@
import re
from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.serializers import UUIDField
from rest_framework.views import APIView
from common.model_runtime_client import verify_request
from common.auth.authenticate import AnonymousAuthentication
from common.result import result
from local_model.runtime import manager
class LocalModelRuntime(APIView):
authentication_classes = [AnonymousAuthentication]
permission_classes = []
def post(self, request):
try:
data = verify_request(request.body, request.headers.get('X-Model-Runtime-Signature'))
except (ValueError, TypeError):
raise PermissionDenied('模型服务认证失败')
model_id = str(UUIDField().run_validation(data.get('model_id')))
operation = data.get('operation')
try:
if operation == 'release':
lease = data.get('lease', '')
if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{32}', lease):
raise ValueError('请求标识无效')
manager.release(model_id, lease)
return result.success({'released': True})
if operation == 'acquire':
lease = manager.acquire(model_id, remote=True)
try:
return result.success({**manager.status(model_id), 'lease': lease})
except Exception:
manager.release(model_id, lease)
raise
if operation == 'load':
manager.submit(model_id)
elif operation == 'unload':
manager.unload(model_id)
elif operation == 'configure':
manager.configure(model_id)
elif operation != 'status':
raise ValueError('模型服务操作无效')
return result.success(manager.status(model_id))
except (ValueError, TimeoutError) as exc:
raise ValidationError(str(exc))

View File

@ -0,0 +1,179 @@
"""Model file assets, independent of runtime/model connection records. Never executes model code."""
import json
import os
import shutil
import stat
import tempfile
import uuid
import zipfile
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
MAX_BYTES = 2 * 1024 ** 3
MAX_FILES = 10000
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
class ImportError(ValueError):
pass
def inspect_model(directory):
root = Path(directory)
if not root.is_dir() or root.is_symlink():
raise ImportError('请选择有效模型目录')
files = []
for current, dirs, names in os.walk(root, followlinks=False):
for name in dirs + names:
if (Path(current) / name).is_symlink():
raise ImportError('模型目录不能包含符号链接')
for name in names:
file = Path(current) / name
if not file.is_file():
raise ImportError('模型目录包含非普通文件')
files.append(file)
if len(files) > MAX_FILES:
raise ImportError('模型文件数量超过限制')
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
if not weights:
raise ImportError('未找到有效模型权重文件')
def read_json(name):
file = root / name
if not file.exists():
return {}
if file.stat().st_size > 2 * 1024 ** 2:
raise ImportError(name + '超过大小限制')
try:
data = json.loads(file.read_text(encoding='utf-8'))
if not isinstance(data, dict):
raise ValueError()
return data
except (ValueError, UnicodeError):
raise ImportError(name + '格式错误')
config = read_json('config.json')
manifest = read_json('model-manifest.json')
quant = config.get('quantization_config') or {}
if not isinstance(quant, dict):
raise ImportError('量化配置格式错误')
quantization = str(quant.get('quant_method', '')).lower()
suffixes = {f.suffix.lower() for f in weights}
if config and suffixes & {'.safetensors', '.bin'}:
if not config.get('model_type') and not config.get('architectures'):
raise ImportError('config.json缺少模型架构信息')
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
elif '.onnx' in suffixes:
format_name = 'ONNX'
elif '.gguf' in suffixes:
format_name = 'GGUF'
elif manifest:
format_name = 'CUSTOM'
else:
raise ImportError('模型目录缺少config.json或model-manifest.json')
# Validate referenced shards without trusting paths supplied by model files.
for index in root.glob('*.index.json'):
mapping = read_json(index.name).get('weight_map')
if not isinstance(mapping, dict) or not mapping:
raise ImportError('权重索引格式错误')
for name in mapping.values():
if not isinstance(name, str) or '\\' in name:
raise ImportError('权重分片路径无效')
relative = PurePosixPath(name)
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
raise ImportError('权重分片缺失或路径无效')
return {
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
'file_count': len(files), 'weight_count': len(weights),
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
'quantization': quantization,
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
'runtime_status': 'NOT_CHECKED',
'runtime_note': '文件已导入,运行能力需接入验证',
}
class ModelImportStore:
def __init__(self, root):
self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
def list(self):
records = []
for path in self.root.glob('*/import.json'):
try:
records.append(json.loads(path.read_text(encoding='utf-8')))
except (ValueError, OSError):
continue
return sorted(records, key=lambda r: r['created_at'], reverse=True)[:200]
def create(self, owner_id, path=None, upload=None):
if (path is None) == (upload is None):
raise ImportError('请选择一种导入方式')
identity = str(uuid.uuid4())
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
try:
if upload is not None:
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
raise ImportError('请上传2GB以内的ZIP模型包')
archive = staging / 'upload.zip'
with archive.open('wb') as output:
size = 0
for chunk in upload.chunks():
size += len(chunk)
if size > MAX_BYTES:
raise ImportError('模型包超过2GB')
output.write(chunk)
source = staging / 'files'
source.mkdir()
try:
with zipfile.ZipFile(archive) as zipped:
members = zipped.infolist()
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
seen = set()
for member in members:
name = PurePosixPath(member.filename)
mode = member.external_attr >> 16
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
or member.filename in seen):
raise ImportError('模型包包含不安全的路径或文件')
seen.add(member.filename)
target = source.joinpath(*name.parts)
if member.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
with zipped.open(member) as incoming, target.open('xb') as output:
shutil.copyfileobj(incoming, output, 1024 * 1024)
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
raise ImportError('模型包损坏或无法解压') from exc
archive.unlink()
# Most model archives contain one enclosing directory.
children = list(source.iterdir())
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
info = inspect_model(model_dir)
final_path = self.root / identity / model_dir.relative_to(staging)
source_type = 'UPLOAD'
display_name = Path(upload.name).stem
else:
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
raise ImportError('请输入服务器绝对目录')
model_dir = Path(path).resolve(strict=True)
info = inspect_model(model_dir)
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
record = {
'id': identity, 'name': display_name, 'path': str(final_path),
'source': source_type, 'owner_id': str(owner_id),
'created_at': datetime.now(timezone.utc).isoformat(),
'status': 'IMPORTED', **info,
}
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
staging.rename(self.root / identity)
return record
except (FileNotFoundError, PermissionError) as exc:
raise ImportError('目录不存在或无访问权限') from exc
finally:
if staging.exists():
shutil.rmtree(staging)

View File

@ -0,0 +1,28 @@
"""补齐已有百炼模型表单中的 TopP;保留全部自定义参数项。"""
from django.db import migrations
TOP_P_FIELD = {
"input_type": "Slider", "label": "TopP", "required": True,
"default_value": 0.8, "relation_show_field_dict": {},
"relation_trigger_field_dict": {}, "trigger_type": "OPTION_LIST",
"attrs": {"min": 0, "max": 1, "step": 0.01, "precision": 2,
"show-input-controls": False, "show-input": True},
"props_info": {}, "field": "top_p",
}
def add_top_p_params_form(apps, schema_editor):
Model = apps.get_model('models_provider', 'Model')
for model in Model.objects.using(schema_editor.connection.alias).filter(
provider='aliyun_bai_lian_model_provider', model_type='LLM'):
form = model.model_params_form or []
if any(item.get('field') == 'top_p' for item in form):
continue
model.model_params_form = [*form, TOP_P_FIELD]
model.save(update_fields=['model_params_form'], using=schema_editor.connection.alias)
class Migration(migrations.Migration):
dependencies = [('models_provider', '0001_initial')]
# 回退代码时保留参数项,不删除可能由管理员维护的数据。
operations = [migrations.RunPython(add_top_p_params_form, migrations.RunPython.noop)]

View File

@ -0,0 +1,109 @@
"""File import regression tests, temporary files only; no database or model runtimes."""
import io
import json
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest.mock import patch
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
class Upload:
def __init__(self, entries):
data = io.BytesIO()
with zipfile.ZipFile(data, 'w') as archive:
for name, content in entries:
archive.writestr(name, content)
self.body = data.getvalue()
self.size, self.name = len(self.body), 'model.zip'
def chunks(self):
yield self.body
class ImportTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.model = self.root / 'model'
self.model.mkdir()
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
self.store = ModelImportStore(self.root / 'imports')
def test_local_path_record_persists_without_copying_or_running(self):
record = self.store.create('owner', path=str(self.model))
self.assertEqual(record['format'], 'HF')
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
self.assertEqual(record['path'], str(self.model.resolve()))
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
def test_quantized_formats(self):
for method in ('gptq', 'awq'):
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
self.assertEqual(inspect_model(self.model)['format'], method.upper())
def test_onnx_gguf_custom(self):
(self.model / 'config.json').unlink()
(self.model / 'model.safetensors').unlink()
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
file = self.model / ('model' + suffix)
file.write_bytes(b'fixture')
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
self.assertEqual(inspect_model(self.model)['format'], expected)
file.unlink()
def test_zip_nested_model_and_persisted_path(self):
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
self.assertEqual(record['source'], 'UPLOAD')
def test_traversal_windows_path_rejected(self):
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
self.assertEqual(self.store.list(), [])
def test_duplicate_archive_entry_rejected(self):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
def test_zip_symlink_rejected(self):
data=io.BytesIO()
with zipfile.ZipFile(data,'w') as archive:
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
archive.writestr(info, '../outside')
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
def test_size_limit_and_cleanup(self):
upload=Upload([('model.onnx', b'x'*100)])
with patch('models_provider.local_imports.MAX_BYTES', 20):
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
self.assertEqual(list(self.store.root.iterdir()), [])
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
(self.model / 'config.json').write_text('{broken')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'config.json').write_text('{"other":1}')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'model.safetensors').unlink()
with self.assertRaises(ImportError):inspect_model(self.model)
def test_missing_or_escaping_shard_rejected(self):
for name in ('absent.safetensors','../outside.bin'):
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
with self.assertRaises(ImportError):inspect_model(self.model)
def test_bad_zip_and_missing_path_leave_no_records(self):
upload=Upload([]);upload.body=b'not zip';upload.size=7
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
self.assertEqual(self.store.list(), [])
def test_exclusive_source(self):
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
if __name__ == '__main__':unittest.main()

View File

@ -0,0 +1,67 @@
import json
from types import SimpleNamespace
from unittest.mock import patch
from django.test import TestCase
from rest_framework.exceptions import ValidationError
from common.constants.permission_constants import RoleConstants
from common.exception.app_exception import AppUnauthorizedFailed
from models_provider.models import Model
from models_provider.views.runtime import ModelLoading
class LoadingApiTests(TestCase):
def setUp(self):
self.model = Model.objects.create(name='loading-test', provider='model_local_provider',
model_type='EMBEDDING', model_name='test-only', credential='test-only', meta={'existing': 'keep'})
self.admin = SimpleNamespace(auth=SimpleNamespace(role_list=[str(RoleConstants.ADMIN.value)], permission_list=[]), data={})
self.view = ModelLoading()
def args(self):
return {'workspace_id': 'default', 'model_id': str(self.model.id)}
def test_policy_is_persistent_and_preserves_existing_metadata(self):
self.admin.data = {'strategy': 'PRELOAD', 'priority': 90}
with patch('models_provider.views.runtime.runtime_request', return_value={'status': 'QUEUED'}) as rpc:
response = self.view.put(self.admin, **self.args())
self.model.refresh_from_db()
self.assertEqual(self.model.meta['existing'], 'keep')
self.assertEqual(self.model.meta['loading_policy'], {'strategy': 'PRELOAD', 'priority': 90, 'idle_seconds': 300})
self.assertEqual(json.loads(response.content)['data']['status'], 'QUEUED')
self.assertEqual(rpc.call_args_list[0].args[0], 'configure')
def test_normal_user_cannot_change_policy_or_load_weights(self):
user = SimpleNamespace(auth=SimpleNamespace(role_list=[], permission_list=[]), data={'action': 'load'})
for method in (self.view.get, self.view.put, self.view.post):
with self.assertRaises(AppUnauthorizedFailed):
method(user, **self.args())
def test_workspace_and_unsupported_provider_are_rejected(self):
with self.assertRaises(ValidationError):
self.view.get(self.admin, workspace_id='other', model_id=str(self.model.id))
self.model.provider = 'aliyun_bai_lian_model_provider'
self.model.model_type = 'LLM'; self.model.save()
response = self.view.get(self.admin, **self.args())
self.assertEqual(json.loads(response.content)['data']['status'], 'EXTERNAL')
with self.assertRaises(ValidationError):
self.view.put(self.admin, **self.args())
def test_arbitrary_actions_and_invalid_policies_do_not_reach_runtime(self):
with patch('models_provider.views.runtime.runtime_request') as rpc:
self.admin.data = {'action': 'shell'}
with self.assertRaises(ValidationError): self.view.post(self.admin, **self.args())
self.admin.data = {'priority': 500}
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
rpc.assert_not_called()
self.model.refresh_from_db()
self.assertNotIn('loading_policy', self.model.meta)
def test_saved_policy_remains_available_when_runtime_is_temporarily_down(self):
self.admin.data = {'strategy': 'PRELOAD', 'priority': 80}
with patch('models_provider.views.runtime.runtime_request', side_effect=ValueError('offline')):
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
response = self.view.get(self.admin, **self.args())
self.model.refresh_from_db()
self.assertEqual(self.model.meta['loading_policy']['priority'], 80)
self.assertEqual(json.loads(response.content)['data']['status'], 'UNAVAILABLE')

View File

@ -0,0 +1,199 @@
import contextlib
import json
import threading
import weakref
import unittest
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from unittest.mock import patch
from common.model_runtime import ModelRuntime, loading_policy
from common.model_runtime_client import signed_request, verify_request
class FakeAdapter:
def __init__(self):
self.loaded, self.unloaded = [], []
self.native_ready = True
self.fail = False
self.block = None
def load(self, spec):
if self.block:
self.block.wait(3)
if self.fail:
raise ValueError('private error details')
self.loaded.append(spec['id'])
self.native_ready = True
return SimpleNamespace(), {'device': 'cpu'}
def unload(self, spec, value):
self.unloaded.append(spec['id'])
self.native_ready = False
def probe(self, spec):
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
def collect(self):
pass
def error(self, exc):
return 'load failed'
class RuntimeTests(unittest.TestCase):
def setUp(self):
self.now = 0
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
for key, priority in [('low', 10), ('high', 90)]}
self.adapter = FakeAdapter()
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
self.adapter, lambda: self.now)
def test_no_load_until_first_inference_and_then_reuses_weights(self):
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
self.assertEqual(self.adapter.loaded, [])
with self.manager.inference('low') as first:
self.assertIsNotNone(first)
with self.manager.inference('low') as second:
self.assertIs(first, second)
self.assertEqual(self.adapter.loaded, ['low'])
def test_startup_preloads_highest_priority_first(self):
self.specs['low']['policy']['strategy'] = 'PRELOAD'
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['low', 'high'])
for key in ['low', 'high']:
self.manager.entries[key]['future'].result(3)
self.assertEqual(self.adapter.loaded, ['high', 'low'])
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
def test_concurrent_first_calls_only_load_once(self):
self.adapter.block = threading.Event()
with ThreadPoolExecutor(max_workers=6) as pool:
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
self.adapter.block.set()
tokens = [future.result(3) for future in futures]
self.assertEqual(len(set(tokens)), 6)
self.assertEqual(self.adapter.loaded, ['low'])
self.assertEqual(self.manager.status('low')['active_requests'], 6)
for token in tokens:
self.manager.release('low', token)
def test_in_use_models_cannot_be_unloaded(self):
token = self.manager.acquire('low')
with self.assertRaises(ValueError):
self.manager.unload('low')
self.manager.release('low', token)
self.manager.unload('low')
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
def test_unload_releases_the_last_loader_reference(self):
class Weights:
pass
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
token = self.manager.acquire('low')
pointer = weakref.ref(self.manager.entries['low']['value'])
self.manager.release('low', token)
self.manager.unload('low')
self.assertIsNone(pointer())
def test_idle_cleanup_respects_active_and_preloaded_models(self):
active = self.manager.acquire('low')
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['high'])
self.manager.entries['high']['future'].result(3)
self.now = 100
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, [])
self.manager.release('low', active)
self.now = 131
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, ['low'])
self.assertEqual(self.manager.status('high')['status'], 'READY')
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
self.adapter.fail = True
with self.assertRaisesRegex(ValueError, '^load failed$'):
self.manager.acquire('low')
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
self.adapter.fail = False
with self.manager.inference('low'):
pass
self.assertEqual(self.manager.status('low')['status'], 'READY')
def test_changed_model_config_never_reuses_old_weights(self):
token = self.manager.acquire('low')
self.specs['low']['fingerprint'] = 'v2'
with self.assertRaises(ValueError):
self.manager.acquire('low')
self.manager.release('low', token)
with self.manager.inference('low'):
pass
self.assertEqual(self.adapter.loaded, ['low', 'low'])
self.assertEqual(self.adapter.unloaded, ['low'])
def test_native_expiration_requires_loading_again(self):
self.specs['low']['driver'] = 'ollama'
token = self.manager.acquire('low')
self.manager.release('low', token)
self.adapter.native_ready = False
token = self.manager.acquire('low')
self.manager.release('low', token)
self.assertEqual(self.adapter.loaded, ['low', 'low'])
def test_policy_validation(self):
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
with self.subTest(value=value), self.assertRaises(ValueError):
loading_policy(value)
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
self.assertEqual(verify_request(body, signature)['operation'], 'load')
with self.assertRaises(ValueError):
verify_request(body.replace(b'load', b'unload'), signature)
with self.assertRaises(ValueError):
verify_request(body, '')
with patch('common.model_runtime_client.time.time', return_value=1):
body, signature = signed_request({'operation': 'load'})
with self.assertRaises(ValueError):
verify_request(body, signature)
class ProviderHookTests(unittest.TestCase):
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
events = []
@contextlib.contextmanager
def lease(model_id):
events.append(('acquire', model_id))
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append(('release', model_id))
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
self.assertEqual(model.embed_query('text'), [1., 2.])
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
self.assertEqual(model._default_params['keep_alive'], -1)
def test_llm_stream_releases_lease_when_generator_is_closed(self):
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
from langchain_ollama import ChatOllama
events = []
@contextlib.contextmanager
def lease(model_id):
events.append('acquire')
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append('release')
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
stream = model._create_chat_stream([])
next(stream); stream.close()
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
self.assertEqual(events, ['acquire', 'release'])

View File

@ -0,0 +1,31 @@
import os
from pathlib import Path
from rest_framework.views import APIView
from common.auth import TokenAuth
from common.auth.authentication import has_permissions
from common.constants.permission_constants import RoleConstants
from common.exception.app_exception import AppApiException
from common.result import result
from maxkb.conf import PROJECT_DIR
from models_provider.local_imports import ModelImportStore, ImportError
class LocalModelImports(APIView):
authentication_classes = [TokenAuth]
@staticmethod
def store():
return ModelImportStore(os.environ.get('MAXKB_MODEL_IMPORT_DIR', str(Path(PROJECT_DIR) / 'model' / 'imports')))
@has_permissions(RoleConstants.ADMIN)
def get(self, request):
return result.success(self.store().list())
@has_permissions(RoleConstants.ADMIN)
def post(self, request):
try:
return result.success(self.store().create(request.user.id,
path=request.data.get('path'), upload=request.FILES.get('file')))
except ImportError as exc:
raise AppApiException(500, str(exc))

View File

@ -0,0 +1,75 @@
from django.db import transaction
from rest_framework.exceptions import ValidationError
from rest_framework.serializers import UUIDField
from rest_framework.views import APIView
from common.auth import TokenAuth
from common.auth.authentication import has_permissions
from common.constants.permission_constants import RoleConstants
from common.model_runtime import loading_policy
from common.model_runtime_client import runtime_request
from common.result import result
from models_provider.models import Model
def managed_model(model):
return ((model.provider == 'model_local_provider' and model.model_type in ('EMBEDDING', 'RERANKER')) or
(model.provider == 'model_ollama_provider' and model.model_type in ('EMBEDDING', 'LLM')))
class ModelLoading(APIView):
authentication_classes = [TokenAuth]
@staticmethod
def model(workspace_id, model_id):
model_id = UUIDField().run_validation(model_id)
model = Model.objects.filter(id=model_id, workspace_id=workspace_id).first()
if model is None:
raise ValidationError('模型不存在')
return model
@staticmethod
def state(model):
policy = loading_policy((model.meta or {}).get('loading_policy'))
if not managed_model(model):
return {**policy, 'supported': False, 'status': 'EXTERNAL'}
try:
return {**runtime_request('status', model.id), 'supported': True}
except ValueError:
return {**policy, 'supported': True, 'status': 'UNAVAILABLE', 'error': '本地模型服务不可用'}
@has_permissions(RoleConstants.ADMIN)
def get(self, request, workspace_id, model_id):
return result.success(self.state(self.model(workspace_id, model_id)))
@has_permissions(RoleConstants.ADMIN)
def put(self, request, workspace_id, model_id):
model = self.model(workspace_id, model_id)
if not managed_model(model):
raise ValidationError('该模型由外部服务运行')
try:
policy = loading_policy(request.data)
except ValueError as exc:
raise ValidationError(str(exc))
with transaction.atomic():
model = Model.objects.select_for_update().get(id=model.id)
model.meta = {**(model.meta or {}), 'loading_policy': policy}
model.save(update_fields=['meta', 'update_time'])
try:
runtime_request('configure', model.id)
except ValueError:
raise ValidationError('策略已保存,本地模型服务暂不可用,请恢复服务后重试')
return result.success(self.state(model))
@has_permissions(RoleConstants.ADMIN)
def post(self, request, workspace_id, model_id):
model = self.model(workspace_id, model_id)
if not managed_model(model):
raise ValidationError('该模型由外部服务运行')
operation = request.data.get('action')
if operation not in ('load', 'unload'):
raise ValidationError('无效操作')
try:
return result.success({**runtime_request(operation, model.id), 'supported': True})
except ValueError as exc:
raise ValidationError(str(exc))

View File

@ -0,0 +1,453 @@
"""AI hub business operations. Secrets never cross the browser boundary."""
from urllib.parse import urlsplit
from django.db import transaction
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
MAX_TEXT = 2_000_000
MAX_INLINE_TEXT = MAX_TEXT + 140_000_000
MAX_FILE = 20 * 1024 * 1024
KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService,
'tasks': AITask, 'artifacts': AIArtifact}
def admin(user):
from common.constants.permission_constants import RoleConstants
return user.role == str(RoleConstants.ADMIN.value)
def require_admin(user):
if not admin(user):
raise PermissionDenied('仅管理员可操作')
def visible(query, user):
# Sharing is explicit, with no implicit public access.
if admin(user):
return query
if query.model in (AIParserResource, AIService):
ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id)
or str(user.id) in obj.allowed_users]
return query.filter(id__in=ids)
return query.filter(owner_id=user.id)
def one(kind, pk, workspace, user, edit=False):
from rest_framework.serializers import UUIDField
pk = UUIDField().run_validation(pk)
model = KINDS.get(kind)
if not model:
raise NotFound()
obj = visible(model.objects.filter(workspace_id=workspace), user).filter(id=pk).first()
if not obj:
raise NotFound('记录不存在或无权访问')
if edit and not admin(user) and str(obj.owner_id) != str(user.id):
raise PermissionDenied('无权修改')
return obj
def text(value, label, required=True, limit=MAX_TEXT):
if not isinstance(value, str) or len(value) > limit or (required and not value.strip()):
raise ValidationError(f'{label}无效')
return value
def integer(value, minimum, maximum, label):
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
raise ValidationError(f'{label}无效')
return value
def users(value):
from users.models import User
if not isinstance(value, list) or len(value) > 500:
raise ValidationError('授权用户无效')
values = {str(item) for item in value}
try:
allowed = set(str(pk) for pk in User.objects.filter(id__in=values, is_active=True).values_list('id', flat=True))
except (ValueError, TypeError):
raise ValidationError('授权用户无效')
# Registration approval is enforced by the existing activation extension.
from users.models.user_manage_ext import UserActivationApplication
pending = set(str(pk) for pk in UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True))
if values - (allowed - pending):
raise ValidationError('只能授权已激活且启用的用户')
return sorted(values)
def authorized_models(workspace, user):
from models_provider.models import Model
from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType
query = Model.objects.filter(workspace_id=workspace, status='SUCCESS')
if not admin(user):
grants = WorkspaceUserResourcePermission.objects.filter(
workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL)
ids = [g.target for g in grants if 'VIEW' in g.permission_list]
query = query.filter(id__in=ids)
return query
def model_for(workspace, user, model_id):
from rest_framework.serializers import UUIDField
model_id = UUIDField().run_validation(model_id)
obj = authorized_models(workspace, user).filter(id=model_id, model_type='LLM').first()
if not obj:
raise PermissionDenied('模型不可用或未授权')
return obj
def serialize(obj, detail=False):
data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()}
if isinstance(obj, AIParserResource):
data.update(enabled=obj.enabled, allowed_users=obj.allowed_users,
options=obj.options, credential_configured=bool(obj.credential))
# Only callers of admin management endpoints receive endpoint addresses.
elif isinstance(obj, AIFlow):
data.update(revision=obj.revision, nodes=obj.nodes)
elif isinstance(obj, AIService):
data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users,
requests_per_minute=obj.requests_per_minute,
key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run')
elif isinstance(obj, AITask):
from system_manage.ai_hub_progress import task_progress
data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt,
started_at=obj.started_at, finished_at=obj.finished_at)
if detail:
data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj))
elif isinstance(obj, AIArtifact):
data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None)
if detail:
data.update(original=obj.original, markdown=obj.markdown, history=obj.history)
return data
def parser_save(workspace, user, data, obj=None):
require_admin(user)
endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/')
parsed = urlsplit(endpoint)
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValidationError('解析服务地址无效')
obj = obj or AIParserResource(workspace_id=workspace, owner_id=user.id)
obj.name = text(data.get('name', obj.name), '名称', limit=128)
obj.endpoint = endpoint
if 'api_key' in data:
from common.utils.rsa_util import rsa_long_encrypt
key = text(data['api_key'], '密钥', required=False, limit=4096)
obj.credential = rsa_long_encrypt(key) if key else ''
enabled = data.get('enabled', obj.enabled)
if not isinstance(enabled, bool):
raise ValidationError('启用状态无效')
obj.enabled = enabled
if 'allowed_users' in data:
obj.allowed_users = users(data['allowed_users'])
options = data.get('options', obj.options)
if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier'}:
raise ValidationError('解析参数无效')
if options.get('protocol', 'legacy') not in ('legacy', 'v1') or options.get('tier', 'standard') not in ('flash', 'basic', 'standard', 'advanced'):
raise ValidationError('解析接口或档位无效')
if options.get('backend', 'pipeline') not in ('pipeline', 'hybrid-auto-engine', 'vlm-auto-engine'):
raise ValidationError('不支持的解析后端')
if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'):
raise ValidationError('识别语言无效')
if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options):
raise ValidationError('解析参数无效')
obj.options = options
obj.save()
return obj
def parser_test(parser):
"""Probe the registered service API, without submitting files or returning service content."""
import time
import requests
from urllib.parse import urlunsplit
from common.utils.rsa_util import rsa_long_decrypt
parsed = urlsplit(parser.endpoint.rstrip('/'))
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password:
raise ValidationError('解析服务地址无效')
base_path = parsed.path
for suffix in ('/v1', '/api'):
if base_path.endswith(suffix):
base_path = base_path[:-len(suffix)]
break
url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', ''))
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
started = time.monotonic()
try:
response = requests.get(url, headers=headers, timeout=(3, 7), allow_redirects=False)
if response.status_code != 200:
raise ValidationError(f'连接测试失败(HTTP {response.status_code})')
data = response.json()
except requests.RequestException as exc:
raise ValidationError('解析服务连接失败或超时') from exc
except ValueError as exc:
raise ValidationError('解析服务未返回有效接口定义') from exc
paths = data.get('paths') if isinstance(data, dict) else None
v1 = parser.options.get('protocol') == 'v1' or parsed.path.endswith('/v1')
suffix = '/tasks' if v1 else '/file_parse'
if not isinstance(paths, dict) or not any(path.endswith(suffix) and 'post' in methods
for path, methods in paths.items() if isinstance(methods, dict)):
raise ValidationError('未识别到对应的解析接口')
return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000),
'message': '解析接口连接正常'}
def validate_nodes(workspace, user, nodes):
if not isinstance(nodes, list) or not 1 <= len(nodes) <= 3:
raise ValidationError('请选择处理节点')
types = [n.get('type') for n in nodes if isinstance(n, dict)]
if len(types) != len(nodes) or types[-1] != 'render' or len(set(types)) != len(types):
raise ValidationError('流程须以导图生成为终点,节点不可重复')
if types != [t for t in ('parse', 'organize', 'render') if t in types]:
raise ValidationError('节点顺序应为解析、整理、导图生成')
result = []
for node in nodes:
kind = node['type']
clean = {'type': kind}
if kind == 'parse':
parser = one('parsers', node.get('resource_id'), workspace, user)
if not parser.enabled:
raise ValidationError('解析服务已停用')
clean['resource_id'] = str(parser.id)
if kind == 'organize':
clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id)
clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000)
temperature = node.get('temperature', 0.2)
if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2:
raise ValidationError('温度无效')
clean['temperature'] = temperature
result.append(clean)
return result
@transaction.atomic
def flow_create(workspace, user, data):
nodes = validate_nodes(workspace, user, data.get('nodes'))
# Every save is immutable; published services always reference a fixed row.
previous = one('flows', data['previous_id'], workspace, user, edit=True) if data.get('previous_id') else None
return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name'), '名称', limit=128), nodes=nodes,
revision=previous.revision + 1 if previous else 1)
@transaction.atomic
def service_create(workspace, user, data):
require_admin(user)
flow = one('flows', data.get('flow_id'), workspace, user)
validate_nodes(workspace, user, flow.nodes)
return AIService.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name'), '名称', limit=128), flow=flow,
allowed_users=users(data.get('allowed_users', [])),
requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率'))
def make_snapshot(workspace, user, nodes):
clean = validate_nodes(workspace, user, nodes)
for node in clean:
if node['type'] == 'parse':
parser = one('parsers', node['resource_id'], workspace, user)
node['options'] = dict(parser.options)
node['resource_version'] = parser.updated_at.isoformat()
if node['type'] == 'organize':
model = model_for(workspace, user, node['model_id'])
node.update(model_name=model.model_name, provider=model.provider, resource_version=model.update_time.isoformat())
return clean
@transaction.atomic
def submit(workspace, user, data, upload=None, service=None):
from users.models import User
current_user = User.objects.select_for_update().get(id=user.id)
if not current_user.is_active:
raise PermissionDenied('用户已停用')
if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
raise ValidationError('待处理任务已达上限')
if service:
if not service.enabled:
raise ValidationError('服务已停用')
nodes = service.flow.nodes
else:
nodes = one('flows', data['flow_id'], workspace, user).nodes if data.get('flow_id') else data.get('nodes')
snapshot = make_snapshot(workspace, user, nodes)
markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT)
if upload and upload.size > MAX_FILE:
raise ValidationError('文件不能超过20MB')
if upload and snapshot[0]['type'] != 'parse':
raise ValidationError('文件任务需要解析节点')
if not upload and snapshot[0]['type'] == 'parse':
raise ValidationError('请选择文件')
if not upload and not markdown.strip():
raise ValidationError('文档内容不能为空')
from system_manage.ai_hub_assets import asset_lock, normalize
with asset_lock():
markdown = text(normalize(markdown, workspace, user), '文档内容', required=False)
return AITask.objects.create(workspace_id=workspace, owner_id=user.id,
name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service,
snapshot={'nodes': snapshot, 'flow_id': str(service.flow_id) if service else data.get('flow_id'),
'flow_revision': service.flow.revision if service else None},
input_data={'markdown': markdown}, file_name=upload.name[:255] if upload else '',
file_data=upload.read() if upload else None)
@transaction.atomic
def artifact_save(workspace, user, data, obj=None):
if obj:
obj = AIArtifact.objects.select_for_update().get(pk=obj.pk)
if data.get('revision') != obj.revision:
raise ValidationError('内容已更新,请重新打开')
history = list(obj.history)
if len(history) >= 30:
raise ValidationError('修订已达30次,请另存为新图谱')
history.append({'revision': obj.revision, 'markdown': obj.markdown, 'layout': obj.layout,
'operator': str(user.id), 'updated_at': obj.updated_at.isoformat()})
obj.history = history
obj.revision += 1
else:
task = one('tasks', data['task_id'], workspace, user) if data.get('task_id') else None
if task and task.status != 'SUCCESS':
raise ValidationError('任务尚未成功')
obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task,
original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT))
obj.name = text(data.get('name'), '名称', limit=128)
obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT)
obj.layout = data.get('layout', 'tree')
if obj.layout not in ('tree', 'radial', 'horizontal', 'vertical'):
raise ValidationError('布局无效')
from system_manage.ai_hub_assets import asset_lock, normalize
with asset_lock():
obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容')
obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False)
for entry in obj.history:
entry['markdown'] = normalize(entry.get('markdown', ''), workspace, user)
obj.save()
return obj
def parse_file(parser, name, content, options=None, progress_callback=None):
import requests
from common.utils.rsa_util import rsa_long_decrypt
opts = options if options is not None else parser.options
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
if opts.get('protocol') == 'v1' or parser.endpoint.rstrip('/').endswith('/v1'):
from system_manage.ai_hub_parser_v1 import parse
return parse(parser, name, content, opts, headers, progress_callback=progress_callback)
try:
response = requests.post(parser.endpoint + '/file_parse', files={'files': (name, content)}, headers=headers,
data={'backend': opts.get('backend', 'pipeline'), 'lang_list': opts.get('language', 'ch'),
'parse_method': 'ocr' if opts.get('ocr') else 'auto', 'return_md': 'true',
'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true',
'formula_enable': str(opts.get('formula', True)).lower(),
'table_enable': str(opts.get('table', True)).lower()}, timeout=(10, 300), allow_redirects=False)
except Exception as exc:
raise ValidationError('解析服务连接失败或超时') from exc
if not 200 <= response.status_code < 300:
raise ValidationError('解析服务调用失败')
try:
payload = response.json()
except ValueError as exc:
raise ValidationError('解析服务返回格式无效') from exc
if not isinstance(payload, dict):
raise ValidationError('解析服务返回格式无效')
entries = payload.get('results', {})
if not isinstance(entries, dict) or len(entries) != 1:
raise ValidationError('解析结果格式无效')
result = next(iter(entries.values()))
if not isinstance(result, dict):
raise ValidationError('解析结果格式无效')
markdown = text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT)
return {'markdown': markdown, 'source': result.get('middle_json', {}), 'parser_id': str(parser.id),
'images': result.get('images', {})}
def markdown_tree(markdown):
"""Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。"""
import re
root = {'id': 'root', 'text': '文档', 'children': []}
heading_stack = [(0, root)]
list_stack = []
fence = None
table_buffer, table_start = [], None
def container():
return list_stack[-1][1] if list_stack else heading_stack[-1][1]
def flush_table():
nonlocal table_buffer, table_start
if not table_buffer:
return
index, lines = table_start, table_buffer
table_buffer, table_start = [], None
def cells(raw):
return [cell.strip() for cell in raw.strip().strip('|').split('|')]
if len(lines) >= 2 and re.match(r'^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$', lines[1]):
headers = cells(lines[0])
aligns = []
for cell in cells(lines[1]):
left, right = cell.startswith(':'), cell.endswith(':')
aligns.append('center' if left and right else 'right' if right else 'left')
rows = [cells(raw) for raw in lines[2:]]
node = {'id': f'n{index}', 'type': 'table', 'text': '表格:' + '、'.join(headers),
'table': {'headers': headers, 'aligns': aligns, 'rows': rows,
'row_count': len(rows), 'column_count': len(headers)},
'children': []}
container()['children'].append(node)
else:
for offset, raw in enumerate(lines):
container()['children'].append({'id': f'n{index + offset}', 'text': raw.strip(), 'children': []})
list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$')
for index, line in enumerate(markdown.splitlines()):
fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
if fence_match:
token = fence_match.group(1)
if fence is None:
fence = token
flush_table()
list_stack.clear()
elif token[0] == fence[0] and len(token) >= len(fence):
fence = None
continue
if fence:
continue
stripped = line.strip()
if stripped.startswith('|') and stripped.count('|') >= 2:
if table_start is None:
table_start = index
list_stack.clear()
table_buffer.append(line)
continue
flush_table()
if not stripped:
continue
heading = re.match(r'^\s{0,3}(#{1,6})\s+(.+)', line)
if heading:
level = len(heading.group(1))
list_stack.clear()
while heading_stack[-1][0] >= level:
heading_stack.pop()
node = {'id': f'n{index}', 'text': heading.group(2).rstrip('# ').strip(), 'children': []}
heading_stack[-1][1]['children'].append(node)
heading_stack.append((level, node))
continue
item = list_pattern.match(line)
if item:
indent = len(item.group(1).expandtabs(4))
while list_stack and list_stack[-1][0] >= indent:
list_stack.pop()
ordered = item.group(3)
node = {'id': f'n{index}', 'type': 'list-item',
'text': item.group(4).strip(),
'list': {'type': 'ol' if ordered else 'ul', 'index': int(ordered) if ordered else None},
'children': []}
container()['children'].append(node)
list_stack.append((indent, node))
continue
if line[:1].isspace() and list_stack:
list_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
continue
list_stack.clear()
heading_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
flush_table()
return root

View File

@ -0,0 +1,213 @@
"""Private, durable image files. References live in existing task/artifact records.
AI_HUB_ASSET_DIR must be on persistent shared storage when Web/worker are on
different hosts. No migration, public static route, or Redis image payloads.
"""
import base64
import hashlib
import io
import json
import os
from pathlib import Path
import re
import time
import uuid
from contextlib import contextmanager
from urllib.parse import quote
from django.conf import settings
from rest_framework.exceptions import NotFound, ValidationError
DATA_IMAGE = re.compile(r'data:image/(?:png|jpe?g|gif|webp|bmp);base64,[A-Za-z0-9+/=\r\n]+', re.I)
ASSET = re.compile(r'/admin/api/workspace/([^/\s"<>]+)/ai_hub/assets/([a-f0-9-]{36})/content')
MAX_IMAGE = 20 * 1024 * 1024
MAX_IMAGES_BYTES = 100 * 1024 * 1024
TTL = 7 * 86400
def root():
return Path(os.environ.get('AI_HUB_ASSET_DIR') or getattr(settings, 'AI_HUB_ASSET_DIR', '')
or Path(__file__).resolve().parents[2] / 'data' / 'ai-hub-assets').resolve()
@contextmanager
def asset_lock():
directory = root()
directory.mkdir(parents=True, exist_ok=True)
lock = directory / '.lock'
stream = lock.open('a+b')
stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0)
deadline = time.monotonic() + 30
while True:
try:
if os.name == 'nt':
import msvcrt
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except OSError:
if time.monotonic() >= deadline:
stream.close()
raise ValidationError('图片存储繁忙,请稍后重试')
time.sleep(.05)
try:
yield
finally:
if os.name == 'nt':
stream.seek(0)
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
else:
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
stream.close()
def _write(path, content):
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
try:
with temporary.open('wb') as stream:
stream.write(content)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _metadata(asset_id):
try:
asset_id = str(uuid.UUID(str(asset_id)))
return json.loads((root() / (asset_id + '.json')).read_text(encoding='utf-8'))
except (OSError, ValueError, TypeError):
raise NotFound('图片不存在或已过期')
def authorize(workspace, user, asset_id):
from system_manage.ai_hub import admin
meta = _metadata(asset_id)
if meta['workspace_id'] != workspace or (meta['owner_id'] != str(user.id) and not admin(user)):
raise NotFound('图片不存在或无权访问')
return meta
def image_bytes(workspace, user, asset_id):
meta = authorize(workspace, user, asset_id)
try:
content = (root() / (str(uuid.UUID(str(asset_id))) + '.bin')).read_bytes()
except OSError:
raise NotFound('图片不存在或已过期')
return content, meta
def _store(uri, workspace, user):
from PIL import Image, UnidentifiedImageError
try:
encoded = uri.split(',', 1)[1]
if len(encoded) > MAX_IMAGE * 4 // 3 + 16:
raise ValueError()
content = base64.b64decode(re.sub(r'\s', '', encoded), validate=True)
if not content or len(content) > MAX_IMAGE:
raise ValueError()
with Image.open(io.BytesIO(content)) as image:
if image.width * image.height > 40_000_000:
raise ValueError()
fmt = image.format
image.verify()
mime = {'PNG': 'image/png', 'JPEG': 'image/jpeg', 'GIF': 'image/gif',
'WEBP': 'image/webp', 'BMP': 'image/bmp'}[fmt]
except (ValueError, KeyError, OSError, UnidentifiedImageError, Image.DecompressionBombError):
raise ValidationError('图片格式无效或超过大小限制')
digest = hashlib.sha256(content).hexdigest()
asset_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f'ai-hub:{workspace}:{user.id}:{digest}'))
meta = {'id': asset_id, 'workspace_id': workspace, 'owner_id': str(user.id),
'sha256': digest, 'mime': mime, 'size': len(content), 'touched_at': time.time()}
if not (root() / (asset_id + '.bin')).exists():
_write(root() / (asset_id + '.bin'), content)
_write(root() / (asset_id + '.json'), json.dumps(meta).encode())
return f'/admin/api/workspace/{quote(workspace, safe="")}/ai_hub/assets/{asset_id}/content'
def normalize(markdown, workspace, user, images=None):
"""Caller holds asset_lock until its database reference is saved."""
replacements = {}
total = 0
def store(uri):
nonlocal total
if uri not in replacements:
total += len(uri) * 3 // 4
if total > MAX_IMAGES_BYTES:
raise ValidationError('文档图片总大小超过限制')
replacements[uri] = _store(uri, workspace, user)
return replacements[uri]
if images:
if not isinstance(images, dict):
raise ValidationError('解析图片格式无效')
# Only replace destinations in Markdown/HTML, never arbitrary words.
for name, uri in images.items():
if not isinstance(name, str) or not isinstance(uri, str) or not DATA_IMAGE.fullmatch(uri):
continue
pattern = re.compile(r'(?P<prefix>\]\(<?|src=["\x27]|^\s*\[[^\]\n]+\]:\s*<?)(?:\./)?(?:images/)?'
+ re.escape(name.removeprefix('images/')) + r'(?=[\s)>"\x27])', re.M)
if pattern.search(markdown):
url = store(uri)
markdown = pattern.sub(lambda m: m['prefix'] + url, markdown)
markdown = DATA_IMAGE.sub(lambda m: store(m[0]), markdown)
for match in ASSET.finditer(markdown):
if match[1] != quote(workspace, safe=''):
raise ValidationError('不能引用其他工作空间的图片')
meta = authorize(workspace, user, match[2])
meta['touched_at'] = time.time()
_write(root() / (match[2] + '.json'), json.dumps(meta).encode())
return markdown
def protect_images(markdown):
"""Keep image destinations out of model rewriting; reject lost/forged tokens."""
images = []
pattern = re.compile(r'!\[[^\]\n]*\]\([^\n]*?\)|<img\b[^>]*>', re.I)
def replace(match):
images.append(match[0])
return f'AIHUBIMAGE{len(images):04d}TOKEN'
return pattern.sub(replace, markdown), images
def restore_images(markdown, images):
# Summarizers may omit or repeat placeholders. The backend owns the image
# manifest, so preserve known positions and append omitted originals once.
known = {str(index + 1): image for index, image in enumerate(images)}
restored = set()
def replace(match):
index = match[1].lstrip('0') or '0'
if index not in known or index in restored:
return ''
restored.add(index)
return known[index]
markdown = re.sub(r'AIHUBIMAGE[\s_*`-]*(\d+)[\s_*`-]*TOKEN', replace, markdown, flags=re.I)
missing = [image for index, image in known.items() if index not in restored]
if missing:
markdown = markdown.rstrip() + '\n\n## 原文图片\n\n' + '\n\n'.join(missing)
return markdown
def cleanup(apply=False, now=None):
"""Only expired, unreferenced Hub images; saved originals/history count as refs."""
from system_manage.models.ai_hub import AIArtifact, AITask
cutoff = (now or time.time()) - TTL
expired = []
with asset_lock():
protected = set()
for artifact in AIArtifact.objects.only('markdown', 'original', 'history').iterator():
content = artifact.markdown + '\n' + artifact.original + '\n' + json.dumps(artifact.history)
protected.update(m[2] for m in ASSET.finditer(content))
for task in AITask.objects.filter(status__in=['PENDING', 'RUNNING']).only('input_data', 'result').iterator():
protected.update(m[2] for m in ASSET.finditer(json.dumps([task.input_data, task.result])))
for path in root().glob('*.json'):
meta = _metadata(path.stem)
if meta['touched_at'] >= cutoff or path.stem in protected:
continue
expired.append(path.stem)
if apply:
(root() / (path.stem + '.bin')).unlink(missing_ok=True)
path.unlink()
return len(expired)

View File

@ -0,0 +1,54 @@
"""Knowledge import uses the same authorized parser and durable task result as mindmaps."""
from django.db import transaction
from django.utils import timezone
from rest_framework.exceptions import PermissionDenied, ValidationError
from knowledge.models import Knowledge
from system_manage import ai_hub as hub
@transaction.atomic
def parser_binding(workspace, knowledge_id, user, data=None):
knowledge = Knowledge.objects.select_for_update().filter(id=knowledge_id, workspace_id=workspace).first()
if not knowledge:
raise ValidationError('知识库不存在')
meta = dict(knowledge.meta or {})
if data is not None:
parser_id = data.get('parser_id')
if parser_id:
parser = hub.one('parsers', parser_id, workspace, user)
if not parser.enabled:
raise ValidationError('解析服务已停用')
meta['parser_id'] = str(parser_id) if parser_id else ''
knowledge.meta = meta
knowledge.save(update_fields=['meta', 'update_time'])
return {'parser_id': meta.get('parser_id', '')}
def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter, limit, overlap):
knowledge = Knowledge.objects.get(id=knowledge_id, workspace_id=workspace)
parser_id = (knowledge.meta or {}).get('parser_id')
if not parser_id:
return None
if user is None:
raise PermissionDenied('解析服务调用需要用户身份')
from system_manage.ai_hub_worker import execute
# Mark as running before commit: the asynchronous consumer cannot claim the same task.
with transaction.atomic():
task = hub.submit(workspace, user, {'name': file.name[:128], 'nodes': [
{'type': 'parse', 'resource_id': parser_id}, {'type': 'render'}]}, upload=file)
task.status = 'RUNNING'
task.started_at = timezone.now()
task.save(update_fields=['status', 'started_at'])
execute(task)
file.seek(0)
if task.status != 'SUCCESS':
raise ValidationError(task.error or '解析失败')
from django.core.files.uploadedfile import SimpleUploadedFile
from common.handle.impl.text.text_split_handle import TextSplitHandle
content = task.result['markdown'].encode('utf-8')
converted = SimpleUploadedFile(file.name + '.md', content)
result = TextSplitHandle().handle(converted, patterns, with_filter, limit,
lambda _: content, lambda _: None, overlap)
result['name'] = file.name
return result, {'resource_id': str(parser_id), 'task_id': str(task.id),
'resource_version': task.snapshot['nodes'][0]['resource_version']}

View File

@ -0,0 +1,59 @@
"""MinerU v1 task API adapter. Only follows paths built from the registered endpoint."""
import re
import time
import requests
from rest_framework.exceptions import ValidationError
def _json(response):
if response.status_code not in (200, 201, 202):
raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})')
try:
body = response.json()
except ValueError as exc:
raise ValidationError('解析服务返回格式无效') from exc
if not isinstance(body, dict):
raise ValidationError('解析服务返回格式无效')
return body
def parse(parser, name, content, options, headers, progress_callback=None):
from system_manage.ai_hub import text, MAX_INLINE_TEXT
endpoint = parser.endpoint.rstrip('/')
deadline = time.monotonic() + 300
try:
response = requests.post(endpoint + '/tasks', files={'files': (name, content)}, headers=headers,
params=[('tier', options.get('tier', 'standard')), ('ocr_mode', 'ocr' if options.get('ocr') else 'auto'),
('output_formats', 'markdown'), ('output_formats', 'middle_json')],
timeout=(10, 30), allow_redirects=False)
task = _json(response)
from system_manage.ai_hub_progress import parser_progress
task_id = task.get('task_id', '')
if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id):
raise ValidationError('解析服务未返回有效任务ID')
if progress_callback:
progress_callback(parser_progress(task))
while task.get('status') in ('queued', 'running'):
if time.monotonic() >= deadline:
raise ValidationError('解析等待超时,后台任务ID:' + task_id)
time.sleep(2)
task = _json(requests.get(endpoint + '/tasks/' + task_id, headers=headers,
timeout=(10, 20), allow_redirects=False))
if progress_callback:
progress_callback(parser_progress(task))
if task.get('status') != 'completed':
raise ValidationError('解析任务未成功,后台任务ID:' + task_id)
result = _json(requests.get(endpoint + '/tasks/' + task_id + '/result', headers=headers,
timeout=(10, 30), allow_redirects=False))
except requests.RequestException as exc:
raise ValidationError('解析服务连接失败或超时') from exc
files = result.get('files', [])
if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) or files[0].get('status') != 'completed':
raise ValidationError('解析文件结果无效')
output = files[0].get('content', {})
if not isinstance(output, dict):
raise ValidationError('解析文件内容无效')
markdown = text(output.get('markdown'), '解析结果', limit=MAX_INLINE_TEXT)
return {'markdown': markdown, 'parser_id': str(parser.id), 'images': output.get('images', {}),
'source': {'task_id': task_id, 'parse': files[0].get('parse'),
'middle_json': output.get('middle_json', {})}}

View File

@ -0,0 +1,37 @@
"""Expose real phases/counts only. A single-file v1 job has no page percentage."""
from django.utils import timezone
LABELS = {'parse': '文档解析中', 'organize': '智能整理中', 'render': '生成思维导图'}
def parser_progress(task):
state = task.get('status')
phase = {'queued': '等待解析服务处理', 'running': '正在解析文档', 'completed': '获取解析结果'}.get(state, '正在解析文档')
raw = task.get('progress') or {}
def count(key):
value = raw.get(key, 0) if isinstance(raw, dict) else 0
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
completed, total = count('completed'), count('total')
percent = min(100, completed * 100 // total) if total > 1 and completed else None
if total > 1:
phase += f'({min(completed, total)}/{total} 个文件)'
return {'stage': phase, 'percent': percent, 'completed_files': completed, 'total_files': total}
def task_progress(task):
elapsed = max(0, int(((task.finished_at or timezone.now()) - (task.started_at or task.created_at)).total_seconds()))
if task.status == 'SUCCESS':
return {'stage': '完成', 'percent': 100, 'elapsed_seconds': elapsed}
if task.status == 'PENDING':
return {'stage': '等待任务处理', 'percent': None, 'elapsed_seconds': elapsed}
if task.status == 'FAILED':
return {'stage': '处理失败', 'percent': None, 'elapsed_seconds': elapsed}
current = next((step for step in task.steps if step.get('status') == 'RUNNING'), {})
detail = current.get('progress') or {}
percent = detail.get('percent')
total_steps = max(1, len(task.snapshot.get('nodes', [])))
complete_steps = sum(step.get('status') == 'SUCCESS' for step in task.steps)
if percent is not None:
percent = min(99, int((complete_steps * 100 + percent) / total_steps))
return {'stage': detail.get('stage') or LABELS.get(current.get('type'), '处理中'),
'percent': percent, 'elapsed_seconds': elapsed}

View File

@ -0,0 +1,104 @@
"""Database-backed single consumer; never consumes legacy Celery queues."""
from django.db import transaction
from django.utils import timezone
from system_manage.models.ai_hub import AITask
from system_manage import ai_hub as hub
def claim():
with transaction.atomic():
task = AITask.objects.select_for_update(skip_locked=True).filter(status='PENDING').order_by('created_at').first()
if not task:
return None
task.status = 'RUNNING'
task.started_at = timezone.now()
task.save(update_fields=['status', 'started_at', 'updated_at'])
return task
def execute(task):
from users.models import User
steps = []
try:
user = User.objects.get(pk=task.owner_id)
if not user.is_active:
raise ValueError('用户已停用')
if task.service_id:
service = hub.one('services', task.service_id, task.workspace_id, user)
if not service.enabled:
raise ValueError('服务已停用')
markdown = task.input_data.get('markdown', '')
original = markdown
source = {}
tree = {}
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree}
task.save(update_fields=['result', 'updated_at'])
for node in task.snapshot['nodes']:
step = {'type': node['type'], 'status': 'RUNNING'}
steps.append(step)
task.steps = steps
task.save(update_fields=['steps', 'updated_at'])
if node['type'] == 'parse':
parser = hub.one('parsers', node['resource_id'], task.workspace_id, user)
if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']:
raise ValueError('解析服务已停用或版本发生变化,请重新提交')
def report_progress(progress):
step['progress'] = progress
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(
steps=steps, updated_at=timezone.now())
report_progress({'stage': '上传文档到解析服务', 'percent': None})
parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress)
from system_manage.ai_hub_assets import asset_lock, normalize
with asset_lock():
markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果')
source = parsed['source']
# Source JSON can contain image data too; preserve structure without inline payloads.
import json
source = json.loads(normalize(json.dumps(source), task.workspace_id, user))
task.result = {'original': original, 'markdown': markdown, 'source': source}
task.save(update_fields=['result', 'updated_at'])
elif node['type'] == 'organize':
from models_provider.tools import get_model, get_model_default_params
from langchain_core.messages import HumanMessage, SystemMessage
model = hub.model_for(task.workspace_id, user, node['model_id'])
if model.update_time.isoformat() != node['resource_version']:
raise ValueError('模型配置已变更,请重新提交')
# Bound input; never silently truncate the original document.
from system_manage.ai_hub_assets import protect_images, restore_images
model_markdown, image_tokens = protect_images(markdown)
if len(model_markdown) > 60000:
raise ValueError('文档超过智能整理长度限制,请分批处理')
try:
instance = get_model(model, **{**get_model_default_params(model),
'temperature': node.get('temperature', 0.2), 'timeout': 180})
prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。'
response = instance.invoke([SystemMessage(content=prompt), HumanMessage(content=model_markdown)])
except Exception as exc:
from rest_framework.exceptions import ValidationError
raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc
markdown = restore_images(hub.text(response.content, '整理结果'), image_tokens)
else:
tree = hub.markdown_tree(markdown)
step['status'] = 'SUCCESS'
task.steps = steps
# Persist intermediate original output so a later model failure does not lose OCR work.
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree}
task.save(update_fields=['steps', 'result', 'updated_at'])
task.status = 'SUCCESS'
except Exception as exc:
if steps:
steps[-1]['status'] = 'FAILED'
# Do not echo URLs, credentials, request bodies or provider exceptions to users/logs.
from rest_framework.exceptions import APIException
if isinstance(exc, APIException):
detail = exc.detail
while isinstance(detail, (list, tuple, dict)) and detail:
detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0]
task.error = str(detail)[:512]
else:
task.error = str(exc)[:512] if isinstance(exc, ValueError) else '执行失败,请检查服务状态后重试'
task.status = 'FAILED'
task.steps = steps
task.finished_at = timezone.now()
task.save(update_fields=['status', 'error', 'steps', 'finished_at', 'updated_at'])
return task

View File

@ -0,0 +1,13 @@
from django.core.management.base import BaseCommand
from system_manage.ai_hub_assets import cleanup
class Command(BaseCommand):
help = 'Check expired Hub images; --apply removes only images not referenced by saved graphs/history.'
def add_arguments(self, parser):
parser.add_argument('--apply', action='store_true')
def handle(self, *args, **options):
count = cleanup(apply=options['apply'])
self.stdout.write(f'{"Removed" if options["apply"] else "Eligible"}: {count}')

View File

@ -0,0 +1,37 @@
import time
from datetime import timedelta
from django.core.management.base import BaseCommand
from django.db import close_old_connections
from django.utils import timezone
from system_manage.models.ai_hub import AITask
from system_manage.ai_hub_worker import claim, execute
class Command(BaseCommand):
help = 'Run the isolated AI hub task consumer (does not run migrations).'
def add_arguments(self, parser):
parser.add_argument('--once', action='store_true')
parser.add_argument('--recover', action='store_true', help='Mark running tasks older than 30 minutes failed')
def handle(self, *args, **options):
if options['recover']:
AITask.objects.filter(status='RUNNING', updated_at__lt=timezone.now() - timedelta(minutes=30)).update(
status='FAILED', error='执行中断,请重试', finished_at=timezone.now())
next_cleanup = time.monotonic() + 86400
while True:
close_old_connections()
if time.monotonic() >= next_cleanup:
from system_manage.ai_hub_assets import cleanup
try:
cleanup(apply=True)
except Exception:
self.stderr.write('图片缓存清理失败,将于下次重试')
next_cleanup = time.monotonic() + 86400
task = claim()
if task:
execute(task)
if options['once']:
return
if not task:
time.sleep(2)

View File

@ -0,0 +1,115 @@
# Generated by Django 5.2.8 on 2026-10-19 10:15
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('system_manage', '0006_accesscontrolpolicy_and_record'),
]
operations = [
migrations.CreateModel(
name='AIFlow',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
('owner_id', models.UUIDField(db_index=True)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('revision', models.PositiveIntegerField(default=1)),
('nodes', models.JSONField(default=list)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='AIParserResource',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
('owner_id', models.UUIDField(db_index=True)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('endpoint', models.URLField(max_length=1024)),
('credential', models.TextField(blank=True, default='')),
('enabled', models.BooleanField(default=True)),
('allowed_users', models.JSONField(default=list)),
('options', models.JSONField(default=dict)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='AIService',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
('owner_id', models.UUIDField(db_index=True)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('enabled', models.BooleanField(default=True)),
('allowed_users', models.JSONField(default=list)),
('key_hash', models.CharField(blank=True, default='', max_length=64)),
('requests_per_minute', models.PositiveIntegerField(default=20)),
('flow', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='system_manage.aiflow')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='AITask',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
('owner_id', models.UUIDField(db_index=True)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('status', models.CharField(db_index=True, default='PENDING', max_length=16)),
('snapshot', models.JSONField(default=dict)),
('input_data', models.JSONField(default=dict)),
('file_name', models.CharField(default='', max_length=255)),
('file_data', models.BinaryField(null=True)),
('result', models.JSONField(default=dict)),
('steps', models.JSONField(default=list)),
('error', models.CharField(default='', max_length=512)),
('attempt', models.PositiveIntegerField(default=1)),
('started_at', models.DateTimeField(null=True)),
('finished_at', models.DateTimeField(null=True)),
('service', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aiservice')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='AIArtifact',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
('owner_id', models.UUIDField(db_index=True)),
('name', models.CharField(max_length=128)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('revision', models.PositiveIntegerField(default=1)),
('original', models.TextField(default='')),
('markdown', models.TextField(default='')),
('layout', models.CharField(default='tree', max_length=32)),
('history', models.JSONField(default=list)),
('task', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aitask')),
],
options={
'abstract': False,
},
),
]

View File

@ -0,0 +1,60 @@
"""Shared AI resources, versioned flows and durable user results."""
import uuid
from django.db import models
class HubBase(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
workspace_id = models.CharField(max_length=128, default='default', db_index=True)
owner_id = models.UUIDField(db_index=True)
name = models.CharField(max_length=128)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
class AIParserResource(HubBase):
endpoint = models.URLField(max_length=1024)
credential = models.TextField(default='', blank=True)
enabled = models.BooleanField(default=True)
allowed_users = models.JSONField(default=list)
options = models.JSONField(default=dict)
class AIFlow(HubBase):
revision = models.PositiveIntegerField(default=1)
nodes = models.JSONField(default=list)
class AIService(HubBase):
flow = models.ForeignKey(AIFlow, on_delete=models.PROTECT)
enabled = models.BooleanField(default=True)
allowed_users = models.JSONField(default=list)
key_hash = models.CharField(max_length=64, default='', blank=True)
requests_per_minute = models.PositiveIntegerField(default=20)
class AITask(HubBase):
service = models.ForeignKey(AIService, null=True, on_delete=models.SET_NULL)
status = models.CharField(max_length=16, default='PENDING', db_index=True)
snapshot = models.JSONField(default=dict)
input_data = models.JSONField(default=dict)
file_name = models.CharField(max_length=255, default='')
file_data = models.BinaryField(null=True)
result = models.JSONField(default=dict)
steps = models.JSONField(default=list)
error = models.CharField(max_length=512, default='')
attempt = models.PositiveIntegerField(default=1)
started_at = models.DateTimeField(null=True)
finished_at = models.DateTimeField(null=True)
class AIArtifact(HubBase):
task = models.ForeignKey(AITask, null=True, on_delete=models.SET_NULL)
revision = models.PositiveIntegerField(default=1)
original = models.TextField(default='')
markdown = models.TextField(default='')
layout = models.CharField(max_length=32, default='tree')
history = models.JSONField(default=list)

View File

@ -0,0 +1,462 @@
import json
from unittest import TestCase
from unittest.mock import patch, Mock
from django.db import transaction
from django.core.files.uploadedfile import SimpleUploadedFile
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
from users.models import User
from users.models.user_manage_ext import UserActivationApplication
from system_manage import ai_hub as hub
from system_manage.ai_hub_worker import claim, execute
from system_manage.models.ai_hub import AIFlow, AIParserResource, AITask, AIArtifact, AIService
class HubTests(TestCase):
def setUp(self):
import tempfile
from django.test import override_settings
from pathlib import Path
self.asset_temp = tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[3] / '.codex_tmp')
self.addCleanup(self.asset_temp.cleanup)
self.asset_settings = override_settings(AI_HUB_ASSET_DIR=self.asset_temp.name)
self.asset_settings.enable()
self.addCleanup(self.asset_settings.disable)
self.atomic = transaction.atomic()
self.atomic.__enter__()
self.admin = User.objects.create(username='admin', nick_name='admin', role='ADMIN', is_active=True)
self.user = User.objects.create(username='user', nick_name='user', role='USER', is_active=True)
self.other = User.objects.create(username='other', nick_name='other', role='USER', is_active=True)
def tearDown(self):
transaction.set_rollback(True)
self.atomic.__exit__(None, None, None)
def parser(self, **kwargs):
return AIParserResource.objects.create(workspace_id='default', owner_id=self.admin.id,
name='测试解析', endpoint='http://parser.test/api', allowed_users=[str(self.user.id)], **kwargs)
def test_parser_grant_and_workspace_isolation(self):
p = self.parser()
self.assertEqual(hub.one('parsers', p.id, 'default', self.user).id, p.id)
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'default', self.other)
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'other', self.admin)
with self.assertRaises(PermissionDenied): hub.one('parsers', p.id, 'default', self.user, edit=True)
def test_only_admin_can_register_parser(self):
with self.assertRaises(PermissionDenied): hub.parser_save('default', self.user, {})
def test_authorization_excludes_rejected_pending_disabled(self):
application = UserActivationApplication.objects.create(user=self.user, status='REJECTED')
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
application.status = 'PENDING'; application.save()
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
application.status = 'APPROVED'; application.save()
self.assertEqual(hub.users([str(self.user.id)]), [str(self.user.id)])
self.user.is_active = False; self.user.save()
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
def test_parser_secret_never_serialized_and_preserved(self):
with patch('common.utils.rsa_util.rsa_long_encrypt', return_value='encrypted'):
p = hub.parser_save('default', self.admin, {'name': '解析', 'endpoint': 'http://parser.test/api', 'api_key': 'private', 'allowed_users': []})
self.assertEqual(p.credential, 'encrypted')
self.assertNotIn('private', json.dumps(hub.serialize(p)))
self.assertNotIn('encrypted', json.dumps(hub.serialize(p)))
p = hub.parser_save('default', self.admin, {'name': '解析2'}, p)
self.assertEqual(p.credential, 'encrypted')
def test_parser_url_and_options_validation(self):
for endpoint in ('file:///tmp/a', 'http://user:secret@server/api', 'http://server/api?key=secret'):
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': endpoint})
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/api', 'options': {'server_url': 'http://other'}})
def test_parser_toggle_preserves_shared_options_and_grants(self):
p = hub.parser_save('default', self.admin, {'name': 'shared', 'endpoint': 'http://server/v1',
'allowed_users': [str(self.user.id)], 'options': {'protocol': 'v1', 'tier': 'basic'}})
self.user.is_active = False
self.user.save(update_fields=['is_active'])
hub.parser_save('default', self.admin, {'enabled': False}, p)
p.refresh_from_db()
self.assertFalse(p.enabled)
self.assertEqual(p.allowed_users, [str(self.user.id)])
self.assertEqual(p.options, {'protocol': 'v1', 'tier': 'basic'})
self.assertEqual(p.name, 'shared')
@patch('requests.get')
def test_parser_connection_checks_protocol_and_does_not_return_service_content(self, get):
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}})
get.return_value = Mock(status_code=200)
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'post': {}}}, 'secret': 'not-for-browser'}
result = hub.parser_test(p)
self.assertTrue(result['connected'])
self.assertNotIn('secret', result)
self.assertEqual(get.call_args.args[0], 'http://server/openapi.json')
self.assertFalse(get.call_args.kwargs['allow_redirects'])
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'get': {}}}}
with self.assertRaises(ValidationError): hub.parser_test(p)
p.endpoint = 'http://server/api'; p.options = {'protocol': 'legacy'}
get.return_value.json.return_value = {'paths': {'/file_parse': {'post': {}}}}
self.assertTrue(hub.parser_test(p)['connected'])
@patch('requests.get')
def test_parser_connection_failures_and_permissions(self, get):
import requests
from types import SimpleNamespace
from system_manage.views.ai_hub import ParserTest
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1'})
with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id)
get.assert_not_called()
get.return_value = Mock(status_code=302)
with self.assertRaises(ValidationError): hub.parser_test(p)
get.return_value = Mock(status_code=200)
get.return_value.json.side_effect = ValueError('sensitive server response')
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
self.assertNotIn('sensitive', str(error.exception))
get.side_effect = requests.Timeout('sensitive URL')
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
self.assertNotIn('sensitive', str(error.exception))
def test_flow_versions_are_immutable(self):
a = hub.flow_create('default', self.user, {'name': '文档', 'nodes': [{'type': 'render'}]})
b = hub.flow_create('default', self.user, {'name': '文档', 'previous_id': str(a.id), 'nodes': [{'type': 'render'}]})
self.assertEqual((a.revision, b.revision), (1, 2))
self.assertNotEqual(a.id, b.id)
def test_invalid_flow_and_ungranted_parser_rejected(self):
p = self.parser()
for nodes in ([{'type': 'render'}, {'type': 'parse', 'resource_id': str(p.id)}], [{'type': 'custom'}], [{'type': 'render'}, {'type': 'render'}]):
with self.assertRaises(ValidationError): hub.validate_nodes('default', self.user, nodes)
with self.assertRaises(NotFound): hub.validate_nodes('default', self.other, [{'type': 'parse', 'resource_id': str(p.id)}, {'type': 'render'}])
def test_submit_claim_and_rule_render(self):
task = hub.submit('default', self.user, {'markdown': '# 标题\n## 内容\n事实42\n```\n# 不作标题\n```', 'nodes': [{'type': 'render'}]})
picked = claim()
self.assertEqual(picked.id, task.id)
self.assertIsNone(claim())
execute(picked); picked.refresh_from_db()
self.assertEqual(picked.status, 'SUCCESS')
tree = picked.result['tree']
self.assertEqual(tree['children'][0]['text'], '标题')
self.assertNotIn('不作标题', json.dumps(tree, ensure_ascii=False))
def test_parser_task_preserves_original_and_source(self):
parser = self.parser()
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('scan.pdf', b'pdf'))
response = Mock(status_code=200)
response.json.return_value = {'results': {'scan': {'md_content': '# 内容\n数值42', 'middle_json': {'pages': [1]}}}}
with patch('requests.post', return_value=response) as post:
execute(claim())
self.assertFalse(post.call_args.kwargs['allow_redirects'])
task.refresh_from_db()
self.assertEqual(task.status, 'SUCCESS')
self.assertEqual(task.result['original'], '# 内容\n数值42')
self.assertEqual(task.result['source']['pages'], [1])
def test_parser_disabled_or_changed_after_submit_fails(self):
parser = self.parser()
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
parser.enabled = False; parser.save()
with patch('requests.post') as post: execute(claim()); post.assert_not_called()
task.refresh_from_db()
self.assertEqual(task.status, 'FAILED')
def test_model_calls_existing_provider_with_selected_parameters(self):
from models_provider.models import Model
model = Model.objects.create(name='整理', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id), 'temperature': 0}, {'type': 'render'}]})
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿')
with patch('models_provider.tools.get_model', return_value=fake) as get:
execute(claim())
self.assertEqual(get.call_args.kwargs['temperature'], 0)
task.refresh_from_db()
self.assertEqual(task.result['original'], '# 原稿')
self.assertEqual(task.result['markdown'], '# 整理稿')
def test_provider_exception_is_sanitized(self):
parser = self.parser()
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
with patch('requests.post', side_effect=RuntimeError('private-url-key')): execute(claim())
task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
self.assertNotIn('private', task.error)
def test_empty_parse_does_not_succeed(self):
parser = self.parser()
response = Mock(status_code=200); response.json.return_value = {'results': {'doc': {'md_content': ''}}}
with patch('requests.post', return_value=response):
with self.assertRaises(ValidationError): hub.parse_file(parser, 'x.pdf', b'pdf')
def test_task_validation_error_is_displayed_without_framework_details(self):
parser = self.parser()
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
with patch('system_manage.ai_hub.parse_file', side_effect=ValidationError('解析失败,请检查文档')):
execute(claim())
task.refresh_from_db()
self.assertEqual(task.error, '解析失败,请检查文档')
def test_artifact_review_preserves_original_and_detects_conflicts(self):
a = hub.artifact_save('default', self.user, {'name': '图谱', 'original': '# OCR原值', 'markdown': '# 复核结果'})
a = hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'original': '篡改原稿', 'markdown': '# 第二版'}, a)
self.assertEqual(a.original, '# OCR原值'); self.assertEqual(a.revision, 2)
self.assertEqual(a.history[0]['markdown'], '# 复核结果')
with self.assertRaises(ValidationError): hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'markdown': '# 旧提交'}, a)
with self.assertRaises(NotFound): hub.one('artifacts', a.id, 'default', self.other)
def test_published_service_is_version_pinned_and_disabled_stops_submit(self):
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
task = hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
self.assertEqual(task.snapshot['flow_revision'], 1)
service.enabled = False; service.save()
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
execute(claim()); task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
def test_unknown_ids_and_missing_inputs_are_validation_errors(self):
with self.assertRaises(ValidationError): hub.one('parsers', None, 'default', self.admin)
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'nodes': [{'type': 'render'}]})
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': 'x', 'nodes': [{'type': 'parse', 'resource_id': str(self.parser().id)}, {'type': 'render'}]})
def api(self, method, kind, data=None, pk=None, user=None, headers=None):
from rest_framework.test import APIRequestFactory, force_authenticate
from system_manage.views.ai_hub import HubView, PublishedRun
request = getattr(APIRequestFactory(), method)('/', data or {}, format='json', **(headers or {}))
force_authenticate(request, user=user or self.admin)
kwargs = {'workspace_id': 'default'}
if kind != 'published': kwargs['kind'] = kind
if pk: kwargs['pk'] = pk
view = PublishedRun if kind == 'published' else HubView
with patch('users.serializers.user.get_workspace_list_by_user', return_value=[{'id': 'default'}]):
response = view.as_view()(request, **kwargs)
return json.loads(response.content) if hasattr(response, 'content') else response.data
def test_api_service_key_rotation_revocation_and_real_task(self):
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
rotated = self.api('post', 'services', {'rotate_key': True}, service.id)['data']
self.assertTrue(rotated['api_key'])
self.assertNotIn('api_key', self.api('get', 'services', pk=service.id)['data'])
denied = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
self.assertNotEqual(denied['code'], 200)
accepted = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user, {'HTTP_X_SERVICE_KEY': rotated['api_key']})
self.assertEqual(accepted['code'], 200)
self.assertEqual(AITask.objects.get(id=accepted['data']['id']).status, 'PENDING')
self.api('post', 'services', {'revoke_key': True}, service.id)
service.refresh_from_db(); self.assertEqual(service.key_hash, '')
def test_api_cannot_edit_shared_service_and_stops_disabled_service(self):
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
response = self.api('post', 'services', {'enabled': False}, service.id, self.user)
self.assertNotEqual(response['code'], 200)
self.api('post', 'services', {'enabled': False}, service.id)
response = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
self.assertNotEqual(response['code'], 200)
def test_api_retry_resets_failure_without_duplicate_task(self):
task = hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
task.status = 'FAILED'; task.error = '中断'; task.save()
response = self.api('post', 'tasks', {}, task.id, self.user)
self.assertEqual(response['code'], 200)
task.refresh_from_db(); self.assertEqual(task.attempt, 2); self.assertEqual(task.status, 'PENDING')
self.assertEqual(AITask.objects.count(), 1)
response = self.api('post', 'tasks', {}, task.id, self.user)
self.assertNotEqual(response['code'], 200)
def test_knowledge_binding_and_shared_parser_execution(self):
from knowledge.models import KnowledgeFolder, Knowledge
from system_manage.ai_hub_knowledge import parser_binding, split_with_parser
KnowledgeFolder.objects.create(id='default', name='root')
knowledge = Knowledge.objects.create(name='测试知识库', desc='', folder_id='default', meta={'segmentation': {'chunk_size': 100}})
parser = self.parser()
parser_binding('default', knowledge.id, self.user, {'parser_id': str(parser.id)})
knowledge.refresh_from_db(); self.assertEqual(knowledge.meta['segmentation']['chunk_size'], 100)
with self.assertRaises(NotFound): parser_binding('default', knowledge.id, self.other, {'parser_id': str(parser.id)})
response = Mock(status_code=200)
response.json.return_value = {'results': {'scan': {'md_content': '# 章节\n识别结果42', 'middle_json': {'pages': [1]}}}}
file = SimpleUploadedFile('scan.pdf', b'pdf')
with patch('requests.post', return_value=response):
result, provenance = split_with_parser('default', knowledge.id, self.user, file, None, False, 100, 20)
task = AITask.objects.get(id=provenance['task_id'])
self.assertEqual(task.status, 'SUCCESS'); self.assertEqual(provenance['resource_id'], str(parser.id))
self.assertIn('识别结果42', json.dumps(result, ensure_ascii=False)); self.assertEqual(file.tell(), 0)
parser_binding('default', knowledge.id, self.user, {'parser_id': ''})
self.assertIsNone(split_with_parser('default', knowledge.id, None, file, None, False, 100, 0))
def test_model_selection_follows_view_grants_instead_of_ownership(self):
from models_provider.models import Model
from types import SimpleNamespace
model = Model.objects.create(name='私有', model_type='LLM', model_name='demo', provider='test', user_id=self.user.id, credential='', status='SUCCESS', model_params_form=[])
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[]):
with self.assertRaises(PermissionDenied): hub.model_for('default', self.user, model.id)
grant = SimpleNamespace(target=str(model.id), permission_list=['VIEW'])
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[grant]):
self.assertEqual(hub.model_for('default', self.user, model.id).id, model.id)
def test_queue_limit_applies_to_published_service_too(self):
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
for _ in range(10):
hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
self.assertEqual(AITask.objects.count(), 10)
def test_stopped_user_cannot_submit_directly(self):
self.user.is_active = False; self.user.save()
with self.assertRaises(PermissionDenied): hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
def test_v1_adapter_polls_registered_endpoint_and_preserves_source(self):
parser = self.parser(options={'protocol': 'v1', 'tier': 'flash'})
created = Mock(status_code=202); created.json.return_value = {'task_id': 'task_test', 'status': 'queued', 'result_url': 'http://untrusted/secret'}
status = Mock(status_code=200); status.json.return_value = {'task_id': 'task_test', 'status': 'completed'}
result = Mock(status_code=200); result.json.return_value = {'files': [{'status': 'completed', 'parse': {'engine': 'native'}, 'content': {'markdown': '# 识别结果42', 'middle_json': {'pages': [1]}}}]}
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=created) as post, patch('system_manage.ai_hub_parser_v1.requests.get', side_effect=[status, result]) as get, patch('system_manage.ai_hub_parser_v1.time.sleep'):
parsed = hub.parse_file(parser, 'test.pdf', b'pdf')
self.assertEqual(parsed['markdown'], '# 识别结果42')
self.assertEqual(parsed['source']['middle_json']['pages'], [1])
self.assertEqual(get.call_args.args[0], parser.endpoint + '/tasks/task_test/result')
self.assertIn(('tier', 'flash'), post.call_args.kwargs['params'])
def test_v1_adapter_rejects_external_task_paths_and_failed_files(self):
parser = self.parser(options={'protocol': 'v1'})
response = Mock(status_code=202); response.json.return_value = {'task_id': '../../other', 'status': 'completed'}
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=response), patch('system_manage.ai_hub_parser_v1.requests.get') as get:
with self.assertRaises(ValidationError): hub.parse_file(parser, 'test.pdf', b'pdf')
get.assert_not_called()
def image_markdown(self):
import base64
import io
from PIL import Image
content = io.BytesIO()
Image.new('RGB', (40, 20), '#2468ac').save(content, 'PNG')
return '# 图片验收\n![结构图](data:image/png;base64,' + base64.b64encode(content.getvalue()).decode() + ')', content.getvalue()
def test_asset_submission_persistence_deduplication_and_isolation(self):
from system_manage.ai_hub_assets import ASSET, image_bytes, root
markdown, content = self.image_markdown()
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
url = ASSET.search(task.input_data['markdown'])
self.assertIsNotNone(url)
self.assertNotIn('base64', task.input_data['markdown'])
self.assertEqual(image_bytes('default', self.user, url[2])[0], content)
with self.assertRaises(NotFound): image_bytes('default', self.other, url[2])
with self.assertRaises(NotFound): image_bytes('other', self.admin, url[2])
with self.assertRaises(NotFound): hub.submit('default', self.other, {'markdown': task.input_data['markdown'], 'nodes': [{'type': 'render'}]})
hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
self.assertEqual(len(list(root().glob('*.bin'))), 1)
def test_asset_cleanup_preserves_original_history_and_inflight(self):
import time
from system_manage.ai_hub_assets import ASSET, cleanup, root
markdown, _ = self.image_markdown()
artifact = hub.artifact_save('default', self.user, {'name': '测试图片', 'markdown': markdown, 'original': '', 'layout': 'tree'})
asset_id = ASSET.search(artifact.markdown)[2]
hub.artifact_save('default', self.user, {'name': artifact.name, 'revision': artifact.revision, 'markdown': '# 删除当前引用'}, artifact)
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
self.assertTrue((root() / (asset_id + '.bin')).exists())
# Only remove the isolated test record; existing shared records are never involved.
artifact.delete()
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
execute(task)
self.assertEqual(cleanup(now=time.time() + 8 * 86400), 1)
self.assertTrue((root() / (asset_id + '.bin')).exists())
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 1)
self.assertFalse((root() / (asset_id + '.bin')).exists())
def test_asset_parser_relative_images_and_invalid_payload(self):
from system_manage.ai_hub_assets import asset_lock, normalize, ASSET
markdown, _ = self.image_markdown()
uri = markdown.split('](', 1)[1][:-1]
with asset_lock():
normalized = normalize('![图](images/a.png)\n<img src="a.png">', 'default', self.user, {'a.png': uri})
self.assertEqual(len(ASSET.findall(normalized)), 2)
with self.assertRaises(ValidationError): normalize('![坏图](data:image/png;base64,aGVsbG8=)', 'default', self.user)
def test_asset_bundle_is_offline_and_model_cannot_drop_images(self):
import io
import zipfile
from types import SimpleNamespace
from system_manage.views.ai_hub import MarkdownExport
from system_manage.ai_hub_assets import protect_images, restore_images
markdown, content = self.image_markdown()
artifact = hub.artifact_save('default', self.user, {'name': '测试导出', 'markdown': markdown})
response = MarkdownExport().post(SimpleNamespace(user=self.user, data={'markdown': artifact.markdown}), 'default')
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
self.assertEqual(len(archive.namelist()), 2)
exported = archive.read('document.md').decode()
self.assertNotIn('/admin/api/', exported)
self.assertIn('images/', exported)
self.assertEqual(archive.read(next(n for n in archive.namelist() if n.startswith('images/'))), content)
protected, images = protect_images(artifact.markdown)
self.assertNotIn('/admin/api/', protected)
self.assertEqual(restore_images(protected, images), artifact.markdown)
self.assertIn(images[0], restore_images('# 整理完成但遗漏图片', images))
self.assertEqual(restore_images(protected + protected, images).count(images[0]), 1)
self.assertIn(images[0], restore_images('AIHUBIMAGE 0001 TOKEN', images))
def test_organize_restores_eight_omitted_images_and_preserves_original(self):
from models_provider.models import Model
markdown, _ = self.image_markdown()
image = markdown.split('\n', 1)[1]
original = '# 报告\n' + '\n\n'.join(image for _ in range(8))
model = Model.objects.create(name='测试整理', model_type='LLM', model_name='demo', provider='test',
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
task = hub.submit('default', self.admin, {'markdown': original, 'nodes': [
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
normalized = task.input_data['markdown']
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理结果\n- 结论42')
with patch('models_provider.tools.get_model', return_value=fake):
execute(claim())
task.refresh_from_db()
self.assertEqual(task.status, 'SUCCESS', task.error)
self.assertEqual(task.result['original'], normalized)
self.assertEqual(task.result['markdown'].count('![结构图]('), 8)
self.assertNotIn('AIHUBIMAGE', task.result['markdown'])
self.assertIn('结论42', task.result['markdown'])
self.assertNotIn('/admin/api/', fake.invoke.call_args.args[0][1].content)
def test_failed_organize_keeps_source_available(self):
from models_provider.models import Model
model = Model.objects.create(name='测试失败', model_type='LLM', model_name='demo', provider='test',
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
task = hub.submit('default', self.admin, {'markdown': '# 保留原稿42', 'nodes': [
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
with patch('models_provider.tools.get_model', side_effect=RuntimeError('private')): execute(claim())
task.refresh_from_db()
self.assertEqual(task.status, 'FAILED')
self.assertEqual(task.result['original'], '# 保留原稿42')
self.assertNotIn('private', task.error)
def test_single_file_progress_is_indeterminate_and_counts_are_real(self):
from system_manage.ai_hub_progress import parser_progress, task_progress
single = parser_progress({'status': 'running', 'progress': {'completed': 0, 'total': 1}})
self.assertIsNone(single['percent'])
self.assertEqual(single['stage'], '正在解析文档')
self.assertEqual(parser_progress({'status': 'queued'})['stage'], '等待解析服务处理')
self.assertEqual(parser_progress({'status': 'running', 'progress': {'completed': 2, 'total': 4}})['percent'], 50)
task = hub.submit('default', self.user, {'markdown': '# 测试', 'nodes': [{'type': 'render'}]})
self.assertIsNone(task_progress(task)['percent'])
task.status = 'RUNNING'; task.steps = [{'type': 'parse', 'status': 'RUNNING', 'progress': single}]
self.assertEqual(hub.serialize(task, True)['progress']['stage'], '正在解析文档')
self.assertIsNone(hub.serialize(task, True)['progress']['percent'])
def test_v1_progress_reaches_database_before_task_finishes(self):
parser = self.parser(options={'protocol': 'v1'})
task = hub.submit('default', self.user, {'nodes': [
{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('test.pdf', b'pdf'))
created = Mock(status_code=202); created.json.return_value = {'task_id': 'test_progress', 'status': 'queued'}
running = Mock(status_code=200); running.json.return_value = {'status': 'running', 'progress': {'completed': 0, 'total': 1}}
done = Mock(status_code=200); done.json.return_value = {'status': 'completed'}
result = Mock(status_code=200); result.json.return_value = {'files': [
{'status': 'completed', 'content': {'markdown': '# 完成42'}}]}
seen = []
responses = iter([running, done, result])
def get(*args, **kwargs):
row = AITask.objects.get(pk=task.pk)
seen.append(hub.serialize(row, True)['progress'])
return next(responses)
with patch('requests.post', return_value=created), patch('requests.get', side_effect=get), patch('system_manage.ai_hub_parser_v1.time.sleep'):
execute(claim())
self.assertEqual([item['stage'] for item in seen], ['等待解析服务处理', '正在解析文档', '获取解析结果'])
self.assertTrue(all(item['percent'] is None for item in seen))
task.refresh_from_db()
self.assertEqual(hub.serialize(task, True)['progress']['percent'], 100)

View File

@ -1,11 +1,13 @@
from django.urls import path
from . import views
from .views.ai_hub import HubView, PublishedRun, ParserTest
from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport
app_name = "system_manage"
# @formatter:off
urlpatterns = [
path('workspace/<str:workspace_id>/ai_hub/exports', MarkdownExport.as_view()),
path('workspace/<str:workspace_id>/ai_hub/assets/<uuid:pk>/content', AssetContent.as_view()),
path('workspace/<str:workspace_id>/ai_hub/parsers/<uuid:pk>/test', ParserTest.as_view()),
path('workspace/<str:workspace_id>/ai_hub/published/<uuid:pk>/run', PublishedRun.as_view()),
path('workspace/<str:workspace_id>/ai_hub/<str:kind>', HubView.as_view()),

View File

@ -0,0 +1,201 @@
import hashlib
import secrets
from django.core.cache import cache
from django.db import transaction
from django.utils import timezone
from rest_framework.views import APIView
from rest_framework.exceptions import ValidationError, PermissionDenied
from common.auth import TokenAuth
from common.result import result
from system_manage import ai_hub as hub
from system_manage.models.ai_hub import AIService, AITask
class HubView(APIView):
authentication_classes = [TokenAuth]
def initial(self, request, *args, **kwargs):
super().initial(request, *args, **kwargs)
from users.serializers.user import get_workspace_list_by_user
if not request.user.is_active or kwargs['workspace_id'] not in {str(w['id']) for w in get_workspace_list_by_user(request.user.id)}:
raise PermissionDenied('无权访问')
def get(self, request, workspace_id, kind, pk=None):
if kind == 'resources':
parsers = hub.visible(hub.AIParserResource.objects.filter(workspace_id=workspace_id, enabled=True), request.user)
from users.models import User
from users.models.user_manage_ext import UserActivationApplication
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
return result.success({'parsers': [hub.serialize(p) for p in parsers],
'models': list(hub.authorized_models(workspace_id, request.user).values('id', 'name', 'model_type')),
'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if hub.admin(request.user) else [],
'is_admin': hub.admin(request.user)})
if pk:
obj = hub.one(kind, pk, workspace_id, request.user)
data = hub.serialize(obj, detail=True)
if kind == 'parsers' and hub.admin(request.user):
data['endpoint'] = obj.endpoint
return result.success(data)
if kind not in hub.KINDS:
raise ValidationError('资源类型无效')
query = hub.visible(hub.KINDS[kind].objects.filter(workspace_id=workspace_id), request.user)
if kind == 'tasks':
query = query.defer('file_data', 'input_data', 'snapshot', 'result')
elif kind == 'artifacts':
query = query.defer('original', 'markdown', 'history')
return result.success([hub.serialize(obj) for obj in query.order_by('-created_at')[:200]])
def post(self, request, workspace_id, kind, pk=None):
data = request.data
if request.FILES:
import json
try:
data = json.loads(request.data.get('data', '{}'))
except (ValueError, TypeError):
raise ValidationError('任务参数无效')
if not isinstance(data, dict):
raise ValidationError('请求参数无效')
if pk:
with transaction.atomic():
obj = hub.one(kind, pk, workspace_id, request.user, edit=True)
if kind == 'parsers':
obj = hub.parser_save(workspace_id, request.user, data, obj)
elif kind == 'artifacts':
obj = hub.artifact_save(workspace_id, request.user, data, obj)
elif kind == 'services':
hub.require_admin(request.user)
obj = AIService.objects.select_for_update().get(pk=obj.pk)
if 'name' in data:
obj.name = hub.text(data['name'], '名称', limit=128)
if 'requests_per_minute' in data:
obj.requests_per_minute = hub.integer(data['requests_per_minute'], 1, 600, '调用频率')
if 'enabled' in data:
if not isinstance(data['enabled'], bool):
raise ValidationError('启用状态无效')
obj.enabled = data['enabled']
if 'allowed_users' in data:
obj.allowed_users = hub.users(data['allowed_users'])
key = secrets.token_urlsafe(32) if data.get('rotate_key') else None
if key:
obj.key_hash = hashlib.sha256(key.encode()).hexdigest()
if data.get('revoke_key'):
obj.key_hash = ''
obj.save()
output = hub.serialize(obj)
if key:
output['api_key'] = key
return result.success(output)
elif kind == 'tasks':
obj = AITask.objects.select_for_update().get(pk=obj.pk)
if obj.status != 'FAILED':
raise ValidationError('仅失败任务可重试')
if hub.make_snapshot(workspace_id, request.user, obj.snapshot['nodes']) != obj.snapshot['nodes']:
raise ValidationError('资源版本已变更,请重新提交任务')
if obj.service_id and not obj.service.enabled:
raise ValidationError('服务已停用')
obj.status, obj.error, obj.steps = 'PENDING', '', []
obj.attempt += 1
obj.started_at = obj.finished_at = None
obj.save()
else:
raise ValidationError('流程版本不可修改,请另存版本')
return result.success(hub.serialize(obj, detail=True))
if kind == 'parsers':
obj = hub.parser_save(workspace_id, request.user, data)
elif kind == 'flows':
obj = hub.flow_create(workspace_id, request.user, data)
elif kind == 'services':
obj = hub.service_create(workspace_id, request.user, data)
elif kind == 'tasks':
# Bound pending backlog per user; GPU work is consumed by the isolated worker.
if AITask.objects.filter(owner_id=request.user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
raise ValidationError('待处理任务已达上限')
obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file'))
elif kind == 'artifacts':
obj = hub.artifact_save(workspace_id, request.user, data)
else:
raise ValidationError('资源类型无效')
return result.success(hub.serialize(obj, detail=True))
class ParserTest(HubView):
http_method_names = ['post', 'options']
def post(self, request, workspace_id, pk):
hub.require_admin(request.user)
parser = hub.one('parsers', pk, workspace_id, request.user)
return result.success(hub.parser_test(parser))
class AssetContent(HubView):
http_method_names = ['get', 'options']
def get(self, request, workspace_id, pk):
from django.http import HttpResponse
from system_manage.ai_hub_assets import image_bytes
content, meta = image_bytes(workspace_id, request.user, pk)
response = HttpResponse(content, content_type=meta['mime'])
response['Cache-Control'] = 'private, no-store'
response['X-Content-Type-Options'] = 'nosniff'
return response
class MarkdownExport(HubView):
http_method_names = ['post', 'options']
def post(self, request, workspace_id):
import io
import zipfile
from django.http import HttpResponse
from system_manage.ai_hub_assets import ASSET, asset_lock, image_bytes, normalize
markdown = hub.text(request.data.get('markdown'), '文档内容')
with asset_lock():
markdown = normalize(markdown, workspace_id, request.user)
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive:
replacements = {}
total = 0
for match in ASSET.finditer(markdown):
if match[0] in replacements:
continue
content, meta = image_bytes(workspace_id, request.user, match[2])
total += len(content)
if total > 100 * 1024 * 1024:
raise ValidationError('导出图片总大小超过限制')
extension = meta['mime'].split('/')[1]
name = f'images/{match[2]}.{extension}'
archive.writestr(name, content)
replacements[match[0]] = name
for url, name in replacements.items():
markdown = markdown.replace(url, name)
archive.writestr('document.md', markdown)
response = HttpResponse(buffer.getvalue(), content_type='application/zip')
response['Content-Disposition'] = 'attachment; filename="markdown.zip"'
return response
class PublishedRun(HubView):
"""Internal service: login AND explicit resource grants; Key is an optional extra credential."""
authentication_classes = [TokenAuth]
http_method_names = ['post', 'options']
def post(self, request, workspace_id, pk):
service = hub.one('services', pk, workspace_id, request.user)
if service.key_hash:
supplied = hashlib.sha256(request.headers.get('X-Service-Key', '').encode()).hexdigest()
if not secrets.compare_digest(supplied, service.key_hash):
raise PermissionDenied('服务Key无效')
minute = int(timezone.now().timestamp()) // 60
key = f'ai-hub-rate:{service.id}:{minute}'
cache.add(key, 0, 70)
if cache.incr(key) > service.requests_per_minute:
raise ValidationError('调用过于频繁')
data = request.data
if request.FILES:
import json
try:
data = json.loads(request.data.get('data', '{}'))
except (ValueError, TypeError):
raise ValidationError('任务参数无效')
task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service)
return result.success(hub.serialize(task))

View File

@ -0,0 +1,48 @@
# Generated for user activation audit and role assignment
import uuid_utils.compat
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('users', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='UserActivationApplication',
fields=[
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
('email', models.EmailField(blank=True, default='', max_length=254, verbose_name='注册邮箱')),
('email_verified', models.BooleanField(default=False, verbose_name='邮箱是否已验证')),
('status', models.CharField(choices=[('PENDING', '待审核'), ('APPROVED', '已通过'), ('REJECTED', '已拒绝')], db_index=True, default='PENDING', max_length=20, verbose_name='审核状态')),
('applied_at', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='申请时间')),
('audit_time', models.DateTimeField(blank=True, null=True, verbose_name='审核时间')),
('reject_reason', models.CharField(blank=True, default='', max_length=500, verbose_name='拒绝原因')),
('audit_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='审核人')),
('user', models.OneToOneField(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='activation_application', to='users.user', verbose_name='关联用户')),
],
options={
'db_table': 'user_activation_application',
'ordering': ['-applied_at'],
},
),
migrations.CreateModel(
name='UserRoleAssignment',
fields=[
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
('role_id', models.CharField(db_index=True, default='USER', max_length=64, verbose_name='角色标识')),
('workspace_id', models.CharField(db_index=True, default='default', max_length=128, verbose_name='工作空间id')),
('granted_at', models.DateTimeField(auto_now_add=True, verbose_name='授权时间')),
('granted_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='授权人')),
('user', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='role_assignments', to='users.user', verbose_name='用户')),
],
options={
'db_table': 'user_role_assignment',
'ordering': ['granted_at'],
'unique_together': {('user', 'role_id', 'workspace_id')},
},
),
]

View File

@ -0,0 +1,57 @@
# coding=utf-8
"""
用户管理扩展模型:
1. UserActivationApplication:初次注册用户的待激活审核申请(独立数据对象)
2. UserRoleAssignment:用户-角色-工作空间授权关系(独立数据对象)
说明:这两个模型用于规格书 2.2.2「设置初次注册用户为激活用户」「更改用户角色」
两个功能过程的独立申报,避免与「启用/停用用户账号」「更改用户信息」共用同一数据对象。
"""
import uuid_utils.compat as uuid
from django.db import models
class ActivationStatus(models.TextChoices):
PENDING = "PENDING", "待审核"
APPROVED = "APPROVED", "已通过"
REJECTED = "REJECTED", "已拒绝"
class UserActivationApplication(models.Model):
"""初次注册用户的激活审核申请。"""
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
user = models.OneToOneField('users.User', on_delete=models.CASCADE, db_constraint=False,
related_name="activation_application", verbose_name="关联用户")
email = models.EmailField(verbose_name="注册邮箱", blank=True, default="")
email_verified = models.BooleanField(default=False, verbose_name="邮箱是否已验证")
status = models.CharField(max_length=20, choices=ActivationStatus.choices,
default=ActivationStatus.PENDING, db_index=True, verbose_name="审核状态")
applied_at = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="申请时间")
audit_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False,
null=True, blank=True, related_name="+", verbose_name="审核人")
audit_time = models.DateTimeField(null=True, blank=True, verbose_name="审核时间")
reject_reason = models.CharField(max_length=500, blank=True, default="", verbose_name="拒绝原因")
class Meta:
db_table = "user_activation_application"
ordering = ["-applied_at"]
class UserRoleAssignment(models.Model):
"""用户-角色-工作空间授权关系。"""
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
user = models.ForeignKey('users.User', on_delete=models.CASCADE, db_constraint=False,
related_name="role_assignments", verbose_name="用户")
role_id = models.CharField(max_length=64, default="USER", db_index=True, verbose_name="角色标识")
workspace_id = models.CharField(max_length=128, default="default", db_index=True, verbose_name="工作空间id")
granted_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False,
null=True, blank=True, related_name="+", verbose_name="授权人")
granted_at = models.DateTimeField(auto_now_add=True, verbose_name="授权时间")
class Meta:
db_table = "user_role_assignment"
unique_together = ("user", "role_id", "workspace_id")
ordering = ["granted_at"]

View File

@ -0,0 +1,113 @@
import json
from rest_framework import serializers
from django.db import IntegrityError, transaction
from django.db.models import Q
from django.utils import timezone
from common.exception.app_exception import AppApiException
from common.utils.common import password_encrypt
from common.utils.rsa_util import decrypt
from users.models import User, UserActivationApplication, ActivationStatus
from users.serializers.user import UserManageSerializer
class RegisterSerializer(UserManageSerializer.UserInstance):
re_password = serializers.CharField(required=True, write_only=True, trim_whitespace=False)
def get_fields(self):
fields = super().get_fields()
for name in ('source', 'pending_activation'):
fields.pop(name)
return fields
def is_valid(self, *, raise_exception=True):
# 重复账号的处理由 register 在锁定申请后按审核状态判断。
return serializers.Serializer.is_valid(self, raise_exception=raise_exception)
def validate(self, attrs):
if attrs['password'] != attrs['re_password']:
raise serializers.ValidationError({'re_password': '两次密码不一致'})
return attrs
@staticmethod
def _match_condition(username: str, email: str) -> Q:
condition = Q(username=username)
if email:
condition = condition | Q(email=email)
return condition
@staticmethod
def _re_register(existing_user: User, data: dict) -> dict:
"""被拒绝的用户修改注册信息后重新提交:清空拒绝原因,回到待审核。"""
condition = RegisterSerializer._match_condition(data['username'], data.get('email', ''))
condition |= Q(nick_name=data['nick_name'])
conflict = User.objects.filter(condition).exclude(id=existing_user.id).exists()
if conflict:
raise AppApiException(500, '用户名、姓名或邮箱已被其他账号占用')
with transaction.atomic():
existing_user.username = data['username']
existing_user.nick_name = data.get('nick_name', '') or existing_user.nick_name
if data.get('email'):
existing_user.email = data['email']
existing_user.phone = data.get('phone', '') or ''
existing_user.password = password_encrypt(data['password'])
existing_user.is_active = False
existing_user.save()
# applied_at 是 auto_now_add,只能用 update 直接改库刷新申请时间
UserActivationApplication.objects.filter(id=existing_user.activation_application.id).update(
status=ActivationStatus.PENDING,
reject_reason='',
email=existing_user.email or '',
email_verified=False,
audit_by=None,
audit_time=None,
applied_at=timezone.now(),
)
return {'status': ActivationStatus.PENDING, 're_register': True}
@staticmethod
@transaction.atomic
def register(instance):
payload = dict(instance)
if payload.get('encryptedData'):
try:
credentials = json.loads(decrypt(payload['encryptedData']))
payload['password'] = credentials['password']
payload['re_password'] = credentials['re_password']
except (ValueError, TypeError, KeyError):
raise AppApiException(500, '注册信息无效')
serializer = RegisterSerializer(data=payload)
serializer.is_valid(raise_exception=True)
data = {key: serializer.validated_data.get(key, '') for key in
('username', 'nick_name', 'email', 'phone', 'password')}
# Public registration never accepts account state, source or role assignments.
data.update(pending_activation=True, source='LOCAL')
existing = User.objects.select_for_update().filter(
RegisterSerializer._match_condition(data['username'], data.get('email', ''))
).first()
if existing is not None:
application = UserActivationApplication.objects.select_for_update().filter(user=existing).first()
if application is None:
raise AppApiException(500, '该用户名或邮箱已存在,请直接登录或联系管理员')
if application.status == ActivationStatus.PENDING:
raise AppApiException(500, '注册申请审核中,请等待管理员审核,请勿重复提交')
if application.status == ActivationStatus.APPROVED:
raise AppApiException(500, '该账号已激活,请直接登录')
if application.status != ActivationStatus.REJECTED:
raise AppApiException(500, '该申请不允许重新提交')
# 公开注册入口不能凭用户名或邮箱覆盖别人的被拒绝账号。
if (existing.username != data['username'] or existing.source != 'LOCAL'
or existing.password != password_encrypt(data['password'])):
raise AppApiException(500, '重新提交请使用原用户名和密码')
# 仅“已拒绝”的申请允许修改注册信息后重新提交
return RegisterSerializer._re_register(existing, data)
if User.objects.filter(nick_name=data['nick_name']).exists():
raise AppApiException(500, '姓名已存在')
try:
UserManageSerializer().save(data, user_id=None, with_valid=False)
except IntegrityError:
raise AppApiException(500, '用户名、姓名或邮箱已存在')
return {'status': ActivationStatus.PENDING}

View File

@ -0,0 +1,224 @@
# coding=utf-8
"""用户管理扩展序列化器:待激活审核、角色授权。"""
from django.core.cache import cache
from django.core.mail import send_mail
from django.core.mail.backends.smtp import EmailBackend
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
from rest_framework import serializers
from common.constants.cache_version import Cache_Version
from common.constants.permission_constants import RoleConstants
from common.exception.app_exception import AppApiException
from common.utils.logger import maxkb_logger
from system_manage.models import SystemSetting, SettingType
from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment
from users.serializers.user import update_user_role
ROLE_OPTIONS = [
{"id": RoleConstants.USER.name, "name": "普通用户"},
{"id": RoleConstants.ADMIN.name, "name": "管理员"},
]
VALID_ROLE_IDS = {item["id"] for item in ROLE_OPTIONS}
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def invalidate_user_auth_cache(user_id):
"""角色/状态变化后清掉权限与角色缓存,下一次请求重新计算。"""
for cache_version in (Cache_Version.PERMISSION_LIST, Cache_Version.ROLE_LIST, Cache_Version.USER):
cache.delete(cache_version.get_key(user_id=user_id), version=cache_version.get_version())
def send_activation_notice(user, approved: bool, reason: str = "") -> bool:
"""激活审核结果邮件通知。
仅在系统设置中配置了发件邮箱(SystemSetting EMAIL)且用户填了邮箱时发送;
发送失败只记录日志,不影响审核结果。
"""
try:
if not user.email:
return False
setting = SystemSetting.objects.filter(type=SettingType.EMAIL.value).first()
meta = (setting.meta or {}) if setting else {}
if not meta.get("email_host"):
return False
name = user.nick_name or user.username
if approved:
subject = "【AI-RAG 知识库平台】账号激活审核通过"
body = (f"您好 {name}:\n\n"
f"您提交的注册申请已审核通过,账号已激活,现在可以登录平台使用。\n\n"
f"如非本人操作,请忽略本邮件。")
else:
subject = "【AI-RAG 知识库平台】账号激活审核未通过"
body = (f"您好 {name}:\n\n"
f"很抱歉,您提交的注册申请未通过审核。\n"
f"拒绝原因:{reason or '不符合激活条件'}\n\n"
f"请在注册页面使用原用户名和密码修改并重新提交资料,审核通过后即可登录。")
connection = EmailBackend(meta.get("email_host"), meta.get("email_port"),
meta.get("email_host_user"), meta.get("email_host_password"),
meta.get("email_use_tls"), False, meta.get("email_use_ssl"), timeout=10)
send_mail(subject, body,
from_email=meta.get("from_email") or meta.get("email_host_user"),
recipient_list=[user.email],
fail_silently=False, connection=connection)
return True
except Exception as exc:
maxkb_logger.error(f"send activation notice failed: user={getattr(user, 'id', '')} {exc}")
return False
class ActivationSerializer(serializers.Serializer):
keyword = serializers.CharField(required=False, allow_blank=True, default="")
status = serializers.CharField(required=False, allow_blank=True, default=ActivationStatus.PENDING)
@staticmethod
def page(current_page: int, page_size: int, keyword: str = "", status: str = ActivationStatus.PENDING):
query_set = UserActivationApplication.objects.select_related("user").all()
if status == 'UNAPPROVED':
query_set = query_set.filter(status__in=[ActivationStatus.PENDING, ActivationStatus.REJECTED])
elif status and status != "ALL":
query_set = query_set.filter(status=status)
if keyword:
query_set = query_set.filter(
Q(user__username__icontains=keyword)
| Q(user__nick_name__icontains=keyword)
| Q(user__email__icontains=keyword)
| Q(user__phone__icontains=keyword)
)
total = query_set.count()
start = max(0, (current_page - 1) * page_size)
records = []
for application in query_set.order_by("-applied_at")[start:start + page_size]:
user = application.user
records.append({
"id": str(application.id),
"user_id": str(user.id),
"username": user.username,
"nick_name": user.nick_name,
"email": user.email or application.email or "",
"phone": user.phone or '',
"email_verified": application.email_verified,
"status": application.status,
"applied_at": _fmt(application.applied_at),
"audit_time": _fmt(application.audit_time),
"reject_reason": application.reject_reason,
})
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
@staticmethod
@transaction.atomic
def approve(application_id: str, audit_user_id: str, role_id: str = RoleConstants.USER.name,
workspace_id: str = "default"):
application = (UserActivationApplication.objects.select_for_update()
.select_related("user").filter(id=application_id).first())
if application is None:
raise AppApiException(500, "待激活申请不存在")
if application.status != ActivationStatus.PENDING:
raise AppApiException(500, "该申请已处理,不能重复审核")
if role_id not in VALID_ROLE_IDS:
raise AppApiException(500, "角色不合法")
application.status = ActivationStatus.APPROVED
application.reject_reason = ""
application.audit_by_id = audit_user_id
application.audit_time = timezone.now()
application.save()
user = application.user
user.is_active = True
user.role = role_id
user.save()
# 审核通过时同步写入角色授权关系,形成独立的授权数据对象
UserRoleAssignment.objects.filter(user_id=user.id).delete()
UserRoleAssignment.objects.create(
user_id=user.id, role_id=role_id,
workspace_id=workspace_id or "default", granted_by_id=audit_user_id
)
invalidate_user_auth_cache(user.id)
transaction.on_commit(lambda: send_activation_notice(user, True))
return {"id": str(application.id), "status": application.status, "user_id": str(user.id)}
@staticmethod
@transaction.atomic
def reject(application_id: str, audit_user_id: str, reason: str = ""):
application = (UserActivationApplication.objects.select_for_update()
.select_related("user").filter(id=application_id).first())
if application is None:
raise AppApiException(500, "待激活申请不存在")
if application.status != ActivationStatus.PENDING:
raise AppApiException(500, "该申请已处理,不能重复审核")
application.status = ActivationStatus.REJECTED
application.audit_by_id = audit_user_id
application.audit_time = timezone.now()
application.reject_reason = reason or "不符合激活条件"
application.save()
application.user.is_active = False
application.user.save()
invalidate_user_auth_cache(application.user.id)
transaction.on_commit(lambda: send_activation_notice(application.user, False, application.reject_reason))
return {"id": str(application.id), "status": application.status, "user_id": str(application.user_id)}
class RoleAssignmentSerializer(serializers.Serializer):
role_id = serializers.CharField(required=True)
workspace_ids = serializers.ListField(child=serializers.CharField(), required=False, allow_empty=True)
@staticmethod
def detail(user_id: str):
user = User.objects.filter(id=user_id).first()
if user is None:
raise AppApiException(500, "用户不存在")
assignments = UserRoleAssignment.objects.filter(user_id=user_id).order_by("granted_at")
records = [{
"role_id": item.role_id,
"workspace_id": item.workspace_id,
"granted_at": _fmt(item.granted_at),
} for item in assignments]
if not records:
records = [{"role_id": user.role, "workspace_id": "default", "granted_at": ""}]
return {
"user": {"id": str(user.id), "username": user.username, "nick_name": user.nick_name,
"email": user.email or "", "role": user.role, "is_active": user.is_active},
"assignments": records,
"available_roles": ROLE_OPTIONS,
}
@staticmethod
@transaction.atomic
def save(user_id: str, role_settings, granted_by_id: str):
user = User.objects.select_for_update().filter(id=user_id).first()
if user is None:
raise AppApiException(500, "用户不存在")
if not isinstance(role_settings, list) or len(role_settings) != 1 or not isinstance(role_settings[0], dict):
raise AppApiException(500, "请选择一个角色")
item = role_settings[0]
role_id = item.get("role_id")
if role_id not in VALID_ROLE_IDS:
raise AppApiException(500, "角色只能为普通用户或管理员")
if item.get('workspace_ids') not in (None, [], ['default']):
raise AppApiException(500, '角色配置无效')
if str(user.id) == 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab':
if role_id != RoleConstants.ADMIN.name:
raise AppApiException(500, '不能修改内置管理员角色')
return RoleAssignmentSerializer.detail(str(user.id))
UserRoleAssignment.objects.filter(user_id=user.id).delete()
UserRoleAssignment.objects.create(
user_id=user.id, role_id=role_id, workspace_id='default', granted_by_id=granted_by_id
)
user.role = role_id
user.save()
update_user_role({'role_setting': [{'role_id': role_id, 'workspace_ids': [
'None' if role_id == RoleConstants.ADMIN.name else 'default'
]}]}, user, granted_by_id)
invalidate_user_auth_cache(user.id)
return RoleAssignmentSerializer.detail(str(user.id))

View File

@ -0,0 +1,15 @@
from drf_spectacular.utils import extend_schema
from rest_framework.permissions import AllowAny
from rest_framework.views import APIView
from common.result import result
from users.serializers.register import RegisterSerializer
class RegisterView(APIView):
authentication_classes = []
permission_classes = [AllowAny]
@extend_schema(summary='用户注册', tags=['User Management'])
def post(self, request):
return result.success(RegisterSerializer.register(request.data))

View File

@ -0,0 +1,73 @@
# coding=utf-8
"""用户管理扩展视图:待激活审核、角色授权。"""
from drf_spectacular.utils import extend_schema
from rest_framework.request import Request
from rest_framework.views import APIView
from common.auth.authenticate import TokenAuth
from common.auth.authentication import has_permissions
from common.constants.permission_constants import PermissionConstants, RoleConstants
from common.log.log import log
from common.result import result
from users.models import ActivationStatus
from users.serializers.user_manage_ext import ActivationSerializer, RoleAssignmentSerializer
class UserActivationPendingView(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="待激活注册申请列表", tags=["User Management"])
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
def get(self, request: Request):
current_page = int(request.query_params.get("current_page", 1))
page_size = int(request.query_params.get("page_size", 20))
keyword = request.query_params.get("keyword", "")
status = request.query_params.get("status", ActivationStatus.PENDING)
return result.success(ActivationSerializer.page(current_page, page_size, keyword, status))
class UserActivationApproveView(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['POST'], summary="通过待激活注册申请", tags=["User Management"])
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
@log(menu='User management', operate='Approve user activation',
get_operation_object=lambda r, k: {'name': k.get('application_id')})
def post(self, request: Request, application_id: str):
role_id = request.data.get("role_id", RoleConstants.USER.name)
workspace_id = request.data.get("workspace_id", "default")
return result.success(
ActivationSerializer.approve(application_id, str(request.user.id), role_id, workspace_id)
)
class UserActivationRejectView(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['POST'], summary="拒绝待激活注册申请", tags=["User Management"])
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
@log(menu='User management', operate='Reject user activation',
get_operation_object=lambda r, k: {'name': k.get('application_id')})
def post(self, request: Request, application_id: str):
reason = request.data.get("reason", "")
return result.success(
ActivationSerializer.reject(application_id, str(request.user.id), reason)
)
class UserRoleAssignmentView(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="查看用户角色授权", tags=["User Management"])
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
def get(self, request: Request, user_id: str):
return result.success(RoleAssignmentSerializer.detail(user_id))
@extend_schema(methods=['PUT'], summary="分配用户角色", tags=["User Management"])
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
@log(menu='User management', operate='Assign user role',
get_operation_object=lambda r, k: {'name': k.get('user_id')})
def put(self, request: Request, user_id: str):
return result.success(
RoleAssignmentSerializer.save(user_id, request.data.get("role_settings", []), str(request.user.id))
)

View File

@ -0,0 +1,20 @@
# 思维导图图片资源
解析任务、手工提交和保存图谱时,将支持的 Base64 位图转换为私有图片文件,Markdown 保存稳定的 `/admin/api/workspace/.../ai_hub/assets/.../content` 地址。历史图谱在再次保存时转换原稿和历史版本中的内联图片。
默认持久化目录为项目根目录 `data/ai-hub-assets`;可通过环境变量 `AI_HUB_ASSET_DIR` 指定。Web 与独立 Hub Worker 必须使用相同目录,多机部署需挂载共享存储。备份应同时包含现有数据库与此目录。增量发布不可覆盖该目录。
文件以工作空间、用户和内容哈希去重,旁路 JSON 保存归属、格式、大小和最后引用时间。无需数据库迁移。图片接口使用现有用户和工作空间鉴权,父页面通过二进制通信供导图预览使用,令牌不进入图片 URL。浏览器仅在当前页面内缓存图片,图片展示可点击放大,失败后可重试。
单图上限 20 MiB、4000 万像素;单次转换图片合计上限 100 MiB。支持 PNG、JPEG、GIF、WebP、BMP。图谱及其原稿、历史版本所引用的图片长期保留;运行中或待处理任务中的图片不会清理。其余资源最后引用七天后可清理,未保存的旧任务图片届时可能失效。
已有独立 Hub Worker 每运行 24 小时执行一次清理,不新增定时进程。手工核验可在原有 Web 环境中执行:
```sh
venv/bin/python scripts/ai_hub_ops.py cleanup-assets
venv/bin/python scripts/ai_hub_ops.py cleanup-assets --apply
```
首条命令只报告数量。清理范围仅限本模块资源目录,不涉及原文档、数据库记录、模型或解析服务文件。
含资源图片的 Markdown 下载为 ZIP,包含 `document.md` 和 `images/`;HTML、导图 SVG 导出会嵌入图片,以便离线打开。智能整理使用图片占位符;重复占位符只恢复一次,遗漏的图片自动补入“原文图片”部分,解析原稿始终保留。

View File

@ -0,0 +1,47 @@
"""Explicit AI hub operations; defaults to a read-only schema check, never auto-migrates."""
import argparse
import os
from pathlib import Path
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=['check', 'plan', 'migrate', 'worker', 'cleanup-assets'], nargs='?', default='check')
parser.add_argument('--apply', action='store_true', help='Apply expired image cleanup (default: report only)')
parser.add_argument('--once', action='store_true')
parser.add_argument('--recover', action='store_true')
args = parser.parse_args()
root = Path(__file__).resolve().parents[1]
os.chdir(root)
sys.path.insert(0, str(root / 'apps'))
os.environ['SERVER_NAME'] = 'web'
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
from django.conf import settings
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
settings.LANGUAGE_CODE = 'zh-hans'
import django
django.setup()
from django.core.management import call_command
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
target = [('system_manage', '0007_ai_hub')]
if args.action in ('plan', 'migrate'):
executor = MigrationExecutor(connection)
plan = executor.migration_plan(target)
if any(backwards for _, backwards in plan):
raise RuntimeError('存在更新迁移,不允许回退,请先核对版本')
call_command('migrate', 'system_manage', '0007_ai_hub', plan=args.action == 'plan', interactive=False)
elif args.action == 'cleanup-assets':
call_command('ai_hub_cleanup_assets', apply=args.apply)
elif args.action == 'worker':
call_command('ai_hub_worker', once=args.once, recover=args.recover)
else:
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
tables = set(connection.introspection.table_names())
for model in (AIParserResource, AIFlow, AIService, AITask, AIArtifact):
print(f'{model._meta.db_table}: {"ready" if model._meta.db_table in tables else "missing"}')
if __name__ == '__main__':
main()

View File

@ -0,0 +1,38 @@
"""Isolated SQLite verification; never connects to the shared PostgreSQL or Redis."""
import importlib
import os
from pathlib import Path
import sys
import unittest
root = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(root / 'apps'))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
from django.apps import AppConfig
from django.conf import settings
settings.LOGGING_CONFIG = None
settings.USE_I18N = False
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
'system_manage', 'models_provider', 'application', 'chat', 'oss')]
import django
django.setup()
if '--migration' in sys.argv or '--migration-check' in sys.argv:
from django.core.management import call_command
call_command('makemigrations', 'system_manage', name='ai_hub',
check='--migration-check' in sys.argv, dry_run='--migration-check' in sys.argv)
sys.exit(0)
from django.db import connection
from users.models import User
from users.models.user_manage_ext import UserActivationApplication
from models_provider.models import Model
from knowledge.models import KnowledgeFolder, Knowledge
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
with connection.schema_editor() as editor:
for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge, AIParserResource, AIFlow, AIService, AITask, AIArtifact):
editor.create_model(model)
suite = unittest.defaultTestLoader.loadTestsFromName('system_manage.test_ai_hub')
outcome = unittest.TextTestRunner(verbosity=2).run(suite)
sys.exit(0 if outcome.wasSuccessful() else 1)

View File

@ -0,0 +1,36 @@
"""Run knowledge metadata and segmentation regression tests against an isolated in-memory database."""
import importlib
import os
from pathlib import Path
import sys
import unittest
root = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(root / 'apps'))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
from django.apps import AppConfig
from django.conf import settings
settings.LOGGING_CONFIG = None
settings.USE_I18N = False
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
'system_manage', 'models_provider', 'application', 'chat', 'oss',
)]
import django
django.setup()
from django.db import connection
from users.models import User, UserActivationApplication
from models_provider.models import Model
from knowledge.models import KnowledgeFolder, Knowledge
with connection.schema_editor() as editor:
for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge):
editor.create_model(model)
suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_knowledge_ext')
result = unittest.TextTestRunner(verbosity=2).run(suite)
sys.exit(0 if result.wasSuccessful() else 1)

View File

@ -0,0 +1,35 @@
"""Run independent index rebuild regression tests against an isolated in-memory database."""
import importlib
import os
from pathlib import Path
import sys
import unittest
root = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(root / 'apps'))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
from django.apps import AppConfig
from django.conf import settings
settings.LOGGING_CONFIG = None
settings.USE_I18N = False
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
'system_manage', 'models_provider', 'application', 'chat', 'oss',
)]
import django
django.setup()
from django.db import connection
from knowledge.models import KnowledgeRebuildTask
with connection.schema_editor() as editor:
editor.create_model(KnowledgeRebuildTask)
suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_rebuild')
result = unittest.TextTestRunner(verbosity=2).run(suite)
sys.exit(0 if result.wasSuccessful() else 1)

View File

@ -0,0 +1,32 @@
"""Exercise the signed API under the actual minimal model-service app configuration."""
import os
from pathlib import Path
import sys
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'apps'))
os.environ['SERVER_NAME'] = 'local_model'
os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0'
from django.conf import settings
settings.configure(SECRET_KEY='test-only', AUTH_HANDLES=[], USE_I18N=False,
INSTALLED_APPS=['django.contrib.contenttypes', 'local_model'],
DATABASES={'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}},
CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}})
import django
django.setup()
from rest_framework.test import APIRequestFactory
from common.model_runtime_client import signed_request
from local_model.views.runtime import LocalModelRuntime
factory = APIRequestFactory()
with patch('common.model_runtime_client._secret', return_value=b'test-only'), patch('local_model.views.runtime.manager') as manager:
manager.status.return_value = {'status': 'UNLOADED'}
body, signature = signed_request({'operation': 'status', 'model_id': '00000000-0000-4000-8000-000000000001'})
response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body,
content_type='application/json', HTTP_X_MODEL_RUNTIME_SIGNATURE=signature))
assert response.status_code == 200, response.content
manager.status.assert_called_once()
manager.reset_mock()
response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body, content_type='application/json'))
assert response.status_code == 403, response.status_code
manager.status.assert_not_called()
print('Minimal model-service API: signed request accepted; unsigned request rejected')

View File

@ -0,0 +1,6 @@
"""Model loading regressions against SQLite in memory; never connects to shared services."""
from pathlib import Path
source = (Path(__file__).with_name('check_knowledge_ext.py')).read_text(encoding='utf-8')
source = source.replace("loadTestsFromName('knowledge.test_knowledge_ext')", "loadTestsFromNames(['models_provider.test_model_runtime', 'models_provider.test_model_loading_api', 'knowledge.test_knowledge_ext'])")
exec(compile(source, __file__, 'exec'))

View File

@ -0,0 +1,36 @@
"""Run account registration regression tests against an isolated in-memory database."""
import importlib
import os
from pathlib import Path
import sys
import unittest
root = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(root / 'apps'))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
from django.apps import AppConfig
from django.conf import settings
settings.LOGGING_CONFIG = None
settings.USE_I18N = False
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
'system_manage', 'models_provider', 'application', 'chat', 'oss',
)]
import django
django.setup()
from django.db import connection
from users.models import User, UserActivationApplication, UserRoleAssignment
with connection.schema_editor() as editor:
for model in (User, UserActivationApplication, UserRoleAssignment):
editor.create_model(model)
suite = unittest.defaultTestLoader.loadTestsFromName('users.tests.RegistrationTests')
result = unittest.TextTestRunner(verbosity=2).run(suite)
sys.exit(0 if result.wasSuccessful() else 1)

View File

@ -0,0 +1,40 @@
"""Single-process development web server; never migrates the shared database."""
import argparse
import os
from pathlib import Path
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--host', default='127.0.0.1')
parser.add_argument('--port', type=int, default=8080)
parser.add_argument('--check', action='store_true', help='Check Django configuration without starting the server')
args = parser.parse_args()
root = Path(__file__).resolve().parents[1]
os.chdir(root)
sys.path.insert(0, str(root / 'apps'))
os.environ['SERVER_NAME'] = 'web'
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
# Redirect local logs before Django imports its logging configuration.
from maxkb import const
const.LOG_DIR = str(root / 'data' / 'dev' / 'logs')
from django.conf import settings
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
settings.LANGUAGE_CODE = 'zh-hans'
import django
django.setup()
from django.core import management
if args.check:
management.call_command('check')
else:
# No autoreloader: avoid loading a second copy of the Python application.
management.call_command('runserver', f'{args.host}:{args.port}', use_reloader=False, insecure_serving=True)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,35 @@
"""Single-process weight service with startup preloads; never runs migrations."""
import argparse
import os
from pathlib import Path
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--host')
parser.add_argument('--port', type=int)
parser.add_argument('--check', action='store_true')
args = parser.parse_args()
root = Path(__file__).resolve().parents[1]
os.chdir(root); sys.path.insert(0, str(root / 'apps'))
os.environ['SERVER_NAME'] = 'local_model'
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
if args.check:
os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0'
from maxkb.const import CONFIG
from django.conf import settings
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
settings.LANGUAGE_CODE = 'zh-hans'
import django
django.setup()
from django.core import management
if args.check:
management.call_command('check')
else:
bind = f"{args.host or CONFIG.get('LOCAL_MODEL_HOST')}:{args.port or CONFIG.get('LOCAL_MODEL_PORT')}"
management.call_command('runserver', bind, use_reloader=False)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,12 @@
import { get, post, request } from '@/request/index'
import useStore from '@/stores'
export const hubBase = () => `/workspace/${useStore().user.getWorkspaceId()}/ai_hub`
export const hubGet = (kind: string, id?: string) => get(`${hubBase()}/${kind}${id ? '/' + id : ''}`)
export const hubPost = (kind: string, data: any, id?: string) => post(`${hubBase()}/${kind}${id ? '/' + id : ''}`, data)
export async function hubBinary(kind: 'assets' | 'exports', id?: string, data?: any) {
const response = await request({ url: `${hubBase()}/${kind}${id ? '/' + id + '/content' : ''}`,
method: kind === 'assets' ? 'get' : 'post', data, responseType: 'blob' })
if (!(response.data instanceof Blob) || response.data.type.includes('json')) throw new Error('图片或导出文件不可用')
return { content: await response.data.arrayBuffer(), mime: response.data.type }
}

View File

@ -0,0 +1,182 @@
<template>
<el-drawer v-model="visible" size="560px" title="索引重建任务" :destroy-on-close="false">
<div v-loading="loading" class="rebuild-detail">
<div class="rebuild-head">
<div>
<div class="rebuild-title">任务 {{ shortId }}</div>
<div class="rebuild-sub">{{ task.config_snapshot?.knowledge_name || '知识库' }}</div>
</div>
<el-tag :type="statusType" effect="light">{{ task.status_label || task.status || '-' }}</el-tag>
</div>
<el-progress
:percentage="task.progress || 0"
:status="progressStatus"
:stroke-width="10"
class="rebuild-progress"
/>
<el-descriptions :column="2" border size="small" class="rebuild-desc">
<el-descriptions-item label="文档总数">{{ task.document_total ?? 0 }}</el-descriptions-item>
<el-descriptions-item label="成功 / 失败">
<span class="color-success">{{ task.document_success ?? 0 }}</span>
/
<span class="color-danger">{{ task.document_failed ?? 0 }}</span>
</el-descriptions-item>
<el-descriptions-item label="向量模型">
{{ task.config_snapshot?.embedding_model_name || task.config_snapshot?.embedding_model || '-' }}
</el-descriptions-item>
<el-descriptions-item label="任务 ID">{{ shortId }}</el-descriptions-item>
<el-descriptions-item label="提交时间">{{ task.created_at || '-' }}</el-descriptions-item>
<el-descriptions-item label="结束时间">{{ task.finished_at || '-' }}</el-descriptions-item>
</el-descriptions>
<el-alert
v-if="task.error_message"
:title="task.error_message"
:type="task.status === 'FAILED' ? 'error' : 'warning'"
:closable="false"
show-icon
class="rebuild-error"
/>
<div class="rebuild-tip">
重建期间检索可能暂不可用。
</div>
</div>
<template #footer>
<el-button @click="visible = false">{{ $t('common.cancel') }}</el-button>
<el-button
v-if="task.status === 'FAILED' || task.status === 'PARTIAL'"
type="primary"
:loading="loading"
@click="retry"
>
重新发起
</el-button>
</template>
</el-drawer>
</template>
<script setup lang="ts">
import { computed, onUnmounted, reactive, ref, watch } from 'vue'
import knowledgeApi from '@/api/knowledge/knowledge'
import { MsgSuccess } from '@/utils/message'
import { t } from '@/locales'
const emit = defineEmits(['refresh'])
const visible = ref(false)
const loading = ref(false)
const knowledgeId = ref('')
const taskId = ref('')
const task = reactive<any>({ config_snapshot: {} })
let timer: any = null
const shortId = computed(() => (task.id ? String(task.id).slice(0, 8) : '-'))
const statusType = computed(() => {
const map: Record<string, string> = {
PENDING: 'info',
RUNNING: 'warning',
SUCCESS: 'success',
PARTIAL: 'warning',
FAILED: 'danger',
}
return map[task.status] || 'info'
})
const progressStatus = computed(() => {
if (task.status === 'SUCCESS') return 'success'
if (task.status === 'FAILED') return 'exception'
return undefined
})
function stopPolling() {
if (timer) {
clearInterval(timer)
timer = null
}
}
function loadTask() {
if (!visible.value || !knowledgeId.value || !taskId.value) return
knowledgeApi.getRebuildTask(knowledgeId.value, taskId.value, loading).then((res: any) => {
Object.assign(task, res.data || {})
if (['SUCCESS', 'FAILED', 'PARTIAL'].includes(task.status)) {
stopPolling()
emit('refresh')
}
})
}
function startPolling() {
stopPolling()
timer = setInterval(loadTask, 2000)
}
function open(id: string, kid: string) {
knowledgeId.value = kid
taskId.value = id
visible.value = true
Object.keys(task).forEach((key) => delete task[key])
startPolling()
loadTask()
}
function retry() {
knowledgeApi.postRetryRebuildTask(knowledgeId.value, taskId.value, loading).then((res: any) => {
const next = res.data
MsgSuccess('已重新发起重建任务')
taskId.value = next.id
Object.assign(task, next)
startPolling()
})
}
watch(visible, (value) => { if (!value) stopPolling() })
onUnmounted(stopPolling)
defineExpose({ open })
</script>
<style scoped>
.rebuild-detail {
min-height: 260px;
}
.rebuild-head {
display: flex;
align-items: flex-start;
justify-content: space-between;
margin-bottom: 16px;
}
.rebuild-title {
font-size: 16px;
font-weight: 600;
color: #1f2329;
}
.rebuild-sub {
font-size: 12.5px;
color: #646a73;
margin-top: 4px;
}
.rebuild-progress {
margin-bottom: 18px;
}
.rebuild-desc {
margin-bottom: 14px;
}
.rebuild-error {
margin-bottom: 12px;
}
.rebuild-tip {
font-size: 12px;
color: #8f959e;
line-height: 1.7;
}
.color-success {
color: #1a9c62;
font-weight: 600;
}
.color-danger {
color: #c9353f;
font-weight: 600;
}
</style>

View File

@ -0,0 +1,158 @@
<template>
<LoginLayout>
<LoginContainer>
<template #logo>
<div class="login-brand">
<ChinaMobileIcon />
<h1 class="login-brand-title">AI-RAG</h1>
</div>
</template>
<h2 class="mb-24">用户注册</h2>
<el-form
ref="formRef"
:model="form"
:rules="rules"
:disabled="submitting"
@submit.prevent="submit"
>
<el-form-item prop="username">
<el-input v-model="form.username" size="large" placeholder="用户名" aria-label="用户名" maxlength="64" autocomplete="username" />
</el-form-item>
<el-form-item prop="nick_name">
<el-input v-model="form.nick_name" size="large" placeholder="姓名" aria-label="姓名" maxlength="64" autocomplete="name" />
</el-form-item>
<el-form-item prop="email">
<el-input v-model="form.email" size="large" placeholder="邮箱" aria-label="邮箱" autocomplete="email" />
</el-form-item>
<el-form-item prop="phone">
<el-input v-model="form.phone" size="large" placeholder="手机号(选填)" aria-label="手机号" maxlength="20" autocomplete="tel" />
</el-form-item>
<el-form-item prop="password">
<el-input v-model="form.password" type="password" size="large" placeholder="密码" aria-label="密码" maxlength="20" autocomplete="new-password" show-password />
</el-form-item>
<el-form-item prop="re_password">
<el-input v-model="form.re_password" type="password" size="large" placeholder="确认密码" aria-label="确认密码" maxlength="20" autocomplete="new-password" show-password />
</el-form-item>
<el-button native-type="submit" type="primary" size="large" class="w-full" :loading="submitting" :disabled="initializing">
注册
</el-button>
</el-form>
<div class="mt-12">
<el-button link type="primary" @click="router.push('/login')">返回登录</el-button>
</div>
</LoginContainer>
</LoginLayout>
</template>
<script setup lang="ts">
import { onBeforeMount, reactive, ref } from 'vue'
import { useRouter } from 'vue-router'
import type { FormInstance, FormRules } from 'element-plus'
import forge from 'node-forge'
import LoginLayout from '@/layout/login-layout/LoginLayout.vue'
import LoginContainer from '@/layout/login-layout/LoginContainer.vue'
import ChinaMobileIcon from '@/components/china-mobile-icon/index.vue'
import UserApi from '@/api/user/user'
import useStore from '@/stores'
import { MsgError, MsgSuccess } from '@/utils/message'
const router = useRouter()
const { user } = useStore()
const formRef = ref<FormInstance>()
const initializing = ref(true)
const submitting = ref(false)
const form = reactive({ username: '', nick_name: '', email: '', phone: '', password: '', re_password: '' })
const rules: FormRules<typeof form> = {
username: [
{ required: true, message: '请输入用户名', trigger: 'blur' },
{ min: 4, max: 64, message: '用户名为 4–64 个字符', trigger: 'blur' },
],
nick_name: [{ required: true, whitespace: true, message: '请输入姓名', trigger: 'blur' }],
email: [
{ required: true, message: '请输入邮箱', trigger: 'blur' },
{ type: 'email', message: '邮箱格式不正确', trigger: 'blur' },
],
phone: [{ pattern: /^1[3-9]\d{9}$/, message: '手机号格式不正确', trigger: 'blur' }],
password: [
{ required: true, message: '请输入密码', trigger: 'blur' },
{
pattern: /^(?=.*[a-z])(?=.*[-_!@#$%^&*`~.()+=])(?:(?=.*[A-Z])|(?=.*\d))[a-zA-Z0-9\-_!@#$%^&*`~.()+=]{6,20}$/,
message: '密码为 6–20 位,含小写字母、特殊字符及数字或大写字母',
trigger: 'blur',
},
],
re_password: [
{ required: true, message: '请确认密码', trigger: 'blur' },
{
validator: (_rule, value, callback) => {
callback(value === form.password ? undefined : new Error('两次密码不一致'))
},
trigger: 'blur',
},
],
}
onBeforeMount(async () => {
try {
await user.asyncGetProfile()
} catch {
MsgError('加载失败,请刷新重试')
} finally {
initializing.value = false
}
})
async function submit() {
if (submitting.value || initializing.value) return
const valid = await formRef.value?.validate().catch(() => false)
if (!valid) return
submitting.value = true
try {
if (!user.rasKey) await user.asyncGetProfile()
const key = forge.pki.publicKeyFromPem(user.rasKey)
const credentials = JSON.stringify({ password: form.password, re_password: form.re_password })
const encryptedData = forge.util.encode64(key.encrypt(forge.util.encodeUtf8(credentials), 'RSAES-PKCS1-V1_5'))
await UserApi.register({
username: form.username.trim(),
nick_name: form.nick_name.trim(),
email: form.email.trim(),
phone: form.phone.trim(),
encryptedData,
})
form.password = ''
form.re_password = ''
MsgSuccess('注册成功,待管理员审核')
await router.replace('/login')
} catch {
// Request errors are displayed by the shared response handler.
} finally {
submitting.value = false
}
}
</script>
<style scoped lang="scss">
.login-brand {
display: flex;
align-items: center;
justify-content: center;
min-height: 48px;
:deep(.china-mobile-icon) {
--china-mobile-icon-width: 236px;
--china-mobile-icon-height: 30px;
--china-mobile-icon-gap: 16px;
}
}
.login-brand-title {
margin: 0;
font-size: 24px;
font-weight: 800;
letter-spacing: 0.8px;
background: linear-gradient(90deg, #6b47e5, #1e71c7);
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
}
:deep(.right-container) {
overflow-y: auto;
}
</style>

View File

@ -0,0 +1,128 @@
<template>
<div class="hub-manager" v-loading="loading">
<el-tabs v-model="tab" @tab-change="load">
<el-tab-pane label="服务" name="services" />
<el-tab-pane label="流程" name="flows" />
<el-tab-pane label="任务" name="tasks" />
<el-tab-pane label="图谱" name="artifacts" />
</el-tabs>
<div class="mb-16">
<el-button v-if="tab === 'flows' || (resources.is_admin && tab === 'services')" type="primary" @click="create">{{ tab === 'services' ? '发布服务' : '新建流程' }}</el-button>
<el-button @click="load">刷新</el-button>
</div>
<el-table :data="rows" row-key="id">
<el-table-column prop="name" label="名称" min-width="180" />
<el-table-column v-if="tab === 'flows' || tab === 'artifacts'" prop="revision" label="版本" width="90" />
<el-table-column v-if="tab === 'tasks'" label="状态" width="120"><template #default="{ row }">{{ statuses[row.status] || row.status }}</template></el-table-column>
<el-table-column v-if="tab === 'services'" label="状态" width="90"><template #default="{ row }">{{ row.enabled ? '启用' : '停用' }}</template></el-table-column>
<el-table-column prop="updated_at" label="更新时间" min-width="190" />
<el-table-column label="操作" min-width="300"><template #default="{ row }">
<el-button link type="primary" @click="open(row)">{{ tab === 'flows' ? '查看 / 新版本' : tab === 'artifacts' ? '打开 / 复核' : '详情' }}</el-button>
<template v-if="tab === 'services' && resources.is_admin">
<el-button link type="primary" @click="toggle(row)">{{ row.enabled ? '停用' : '启用' }}</el-button>
<el-button link type="primary" @click="rotate(row)">更新Key</el-button>
<el-button v-if="row.key_configured" link @click="revoke(row)">撤销Key</el-button>
</template>
<el-button v-if="tab === 'tasks' && row.status === 'FAILED'" link type="primary" @click="retry(row)">重试</el-button>
</template></el-table-column>
</el-table>
<el-dialog v-model="dialog" :title="title" width="740px" destroy-on-close>
<el-form label-position="top">
<el-form-item v-if="tab !== 'tasks'" label="名称"><el-input v-model="form.name" maxlength="128" /></el-form-item>
<template v-if="tab === 'flows'">
<el-form-item label="文档解析"><el-select v-model="form.parser_id" clearable placeholder="不解析文件"><el-option v-for="p in resources.parsers" :key="p.id" :label="p.name" :value="p.id"/></el-select></el-form-item>
<el-form-item label="智能整理"><el-select v-model="form.model_id" clearable placeholder="按原文生成"><el-option v-for="m in llms" :key="m.id" :label="m.name" :value="m.id"/></el-select></el-form-item>
<el-form-item v-if="form.model_id" label="提示词"><el-input v-model="form.prompt" type="textarea" :rows="4" /></el-form-item>
<el-form-item v-if="form.model_id" label="温度"><el-input-number v-model="form.temperature" :min="0" :max="2" :step="0.1" /></el-form-item>
<el-form-item label="输出">思维导图</el-form-item>
</template>
<template v-if="tab === 'services'">
<el-form-item label="流程版本"><el-select v-model="form.flow_id" :disabled="!!form.id"><el-option v-for="f in flows" :key="f.id" :label="`${f.name} · v${f.revision}`" :value="f.id" /></el-select></el-form-item>
<el-form-item v-if="!form.id" label="每分钟调用次数"><el-input-number v-model="form.requests_per_minute" :min="1" :max="600"/></el-form-item>
<el-form-item v-if="form.id" label="服务接口"><el-input :model-value="serviceUrl(form)" readonly /></el-form-item>
<el-form-item v-if="form.id" label="调用"><el-input v-model="runMarkdown" type="textarea" :rows="3" placeholder="Markdown"/><el-input v-model="runKey" type="password" show-password placeholder="服务Key(如已启用)"/><input type="file" @change="runFile = ($event.target as HTMLInputElement).files?.[0] || null"/><el-button @click="runService">提交任务</el-button></el-form-item>
</template>
<el-form-item v-if="resources.is_admin && tab === 'services'" label="授权用户">
<el-select v-model="form.allowed_users" multiple filterable><el-option v-for="u in resources.users || []" :key="u.id" :label="u.name" :value="u.id"/></el-select>
</el-form-item>
<template v-if="tab === 'tasks'">
<el-descriptions :column="1" border><el-descriptions-item label="任务ID">{{ form.id }}</el-descriptions-item><el-descriptions-item label="状态">{{ statuses[form.status] }}</el-descriptions-item><el-descriptions-item label="执行次数">{{ form.attempt }}</el-descriptions-item><el-descriptions-item v-if="form.error" label="失败原因">{{ form.error }}</el-descriptions-item></el-descriptions>
<el-table :data="form.steps || []"><el-table-column label="节点"><template #default="{ row }">{{ nodeLabels[row.type] }}</template></el-table-column><el-table-column label="状态"><template #default="{ row }">{{ statuses[row.status] }}</template></el-table-column></el-table>
<el-form-item label="解析原稿"><el-input :model-value="form.result?.original" type="textarea" :rows="5" readonly /></el-form-item>
<el-form-item label="复核内容"><el-input v-model="reviewMarkdown" type="textarea" :rows="8" /></el-form-item>
<el-button v-if="form.status === 'SUCCESS'" type="primary" @click="saveTaskResult">确认保存图谱</el-button>
</template>
<template v-if="tab === 'artifacts'">
<el-form-item label="原始识别内容"><el-input :model-value="form.original" type="textarea" :rows="5" readonly/></el-form-item>
<el-form-item label="修订内容"><el-input v-model="form.markdown" type="textarea" :rows="10"/></el-form-item>
<el-form-item label="布局"><el-select v-model="form.layout"><el-option v-for="(name, value) in layouts" :key="value" :label="name" :value="value"/></el-select></el-form-item>
<el-form-item label="历史修订"><el-select placeholder="查看历史" @change="showHistory"><el-option v-for="h in form.history || []" :key="h.revision" :label="`v${h.revision}`" :value="h.revision"/></el-select></el-form-item>
<el-button @click="emit('open-artifact', form); dialog = false">在工作台打开</el-button>
</template>
</el-form>
<template #footer><el-button @click="dialog = false">关闭</el-button><el-button v-if="canSave" type="primary" :loading="saving" @click="save">{{ tab === 'flows' && form.id ? '保存新版本' : '保存' }}</el-button></template>
</el-dialog>
</div>
</template>
<script setup lang="ts">
import { computed, onMounted, ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import { hubGet, hubPost, hubBase } from '@/api/ai-hub'
import { request } from '@/request/index'
const emit = defineEmits(['open-artifact'])
const tab = ref('services'), loading = ref(false), saving = ref(false), dialog = ref(false)
const rows = ref<any[]>([]), flows = ref<any[]>([]), form = ref<any>({}), resources = ref<any>({ parsers: [], models: [] })
const reviewMarkdown = ref(''), runMarkdown = ref(''), runKey = ref(''), runFile = ref<File | null>(null)
const llms = computed(() => resources.value.models.filter((m: any) => m.model_type === 'LLM'))
const statuses: Record<string, string> = { PENDING: '待处理', RUNNING: '处理中', SUCCESS: '成功', FAILED: '失败' }
const nodeLabels: Record<string, string> = { parse: '文档解析', organize: '智能整理', render: '导图生成' }
const layouts = { tree: '树状' }
const title = computed(() => ({ services: '导图服务', flows: '处理流程', tasks: '任务详情', artifacts: '图谱复核' })[tab.value as 'services'])
const canSave = computed(() => ['flows', 'artifacts'].includes(tab.value) || (resources.value.is_admin && tab.value === 'services'))
async function load() {
loading.value = true
try {
const [r, list, f] = await Promise.all([hubGet('resources'), hubGet(tab.value), hubGet('flows')])
resources.value = r.data; rows.value = list.data; flows.value = f.data
} finally { loading.value = false }
}
function create() { form.value = { name: '', allowed_users: [], enabled: true, parser_id: '', model_id: '', prompt: '保留原文事实、数值和语言,整理为分级Markdown标题和要点。', temperature: 0.2, requests_per_minute: 20 }; dialog.value = true }
async function open(row: any) {
form.value = (await hubGet(tab.value, row.id)).data
if (tab.value === 'flows') {
const p = form.value.nodes.find((n: any) => n.type === 'parse'), m = form.value.nodes.find((n: any) => n.type === 'organize')
Object.assign(form.value, { parser_id: p?.resource_id || '', model_id: m?.model_id || '', prompt: m?.prompt || '', temperature: m?.temperature ?? 0.2 })
}
reviewMarkdown.value = form.value.result?.markdown || ''; runMarkdown.value = ''; runKey.value = ''; runFile.value = null
dialog.value = true
}
async function save() {
saving.value = true
try {
let body: any = { ...form.value }, id = form.value.id
if (tab.value === 'flows') {
body = { name: body.name, previous_id: id, nodes: [] }; id = undefined
if (form.value.parser_id) body.nodes.push({ type: 'parse', resource_id: form.value.parser_id })
if (form.value.model_id) body.nodes.push({ type: 'organize', model_id: form.value.model_id, prompt: form.value.prompt, temperature: form.value.temperature })
body.nodes.push({ type: 'render' })
}
await hubPost(tab.value, body, id); dialog.value = false; ElMessage.success('已保存'); await load()
} finally { saving.value = false }
}
async function toggle(row: any) { await hubPost('services', { enabled: !row.enabled }, row.id); await load() }
async function retry(row: any) { await hubPost('tasks', {}, row.id); await load() }
async function rotate(row: any) { await ElMessageBox.confirm('更新后原Key失效', '更新Key'); const r = await hubPost('services', { rotate_key: true }, row.id); await ElMessageBox.alert(r.data.api_key, '请保存Key'); await load() }
async function revoke(row: any) { await ElMessageBox.confirm('撤销后使用登录权限访问', '撤销Key'); await hubPost('services', { revoke_key: true }, row.id); await load() }
function serviceUrl(row: any) { return `${location.origin}${import.meta.env.BASE_URL}api${hubBase()}/published/${row.id}/run` }
async function runService() {
const data = new FormData(); data.append('data', JSON.stringify({ markdown: runMarkdown.value }))
if (runFile.value) data.append('file', runFile.value)
await request({ url: `${hubBase()}/published/${form.value.id}/run`, method: 'post', data, headers: { 'X-Service-Key': runKey.value }, timeout: 30000 })
ElMessage.success('任务已提交'); dialog.value = false; tab.value = 'tasks'; await load()
}
async function saveTaskResult() { await hubPost('artifacts', { name: form.value.name, task_id: form.value.id, markdown: reviewMarkdown.value, layout: 'tree' }); ElMessage.success('已保存'); dialog.value = false; tab.value = 'artifacts'; await load() }
function showHistory(revision: number) { const h = form.value.history.find((item: any) => item.revision === revision); if (h) ElMessageBox.alert(h.markdown, `v${revision}`, { customStyle: { maxWidth: '80vw', whiteSpace: 'pre-wrap', maxHeight: '80vh', overflow: 'auto' } }) }
onMounted(load)
</script>
<style scoped>.hub-manager { padding: 16px 24px; height: 100%; overflow: auto; } .el-select { width: 100%; }</style>

View File

@ -20,13 +20,13 @@ import { ref, onMounted, onBeforeUnmount } from 'vue'
import { useRoute } from 'vue-router'
import { ElMessage } from 'element-plus'
import systemConfigApi from '@/api/system-settings/system-config'
import { hubGet, hubPost } from '@/api/ai-hub'
import { hubGet, hubPost, hubBinary } from '@/api/ai-hub'
const mindmapUrl = ref<string>('')
const loading = ref(false)
const route = useRoute()
const frame = ref<HTMLIFrameElement>()
const allowedKinds = new Set(['resources', 'tasks', 'artifacts'])
const allowedKinds = new Set(['resources', 'tasks', 'artifacts', 'assets', 'exports'])
function childOrigin() { try { return new URL(mindmapUrl.value, location.href).origin } catch { return '' } }
async function openRequestedArtifact() {
const id = route.query.artifact
@ -43,6 +43,12 @@ async function handleBridge(event: MessageEvent) {
const { id, kind, recordId, method, data, file } = event.data
if (typeof id !== 'string' || id.length > 128 || !allowedKinds.has(kind) || !['get', 'post'].includes(method) || (recordId && !/^[a-f0-9-]{36}$/i.test(recordId))) return
try {
if (kind === 'assets' || kind === 'exports') {
if ((kind === 'assets' && (method !== 'get' || !recordId)) || (kind === 'exports' && (method !== 'post' || recordId))) throw new Error('请求无效')
const binary = await hubBinary(kind, recordId, data)
frame.value?.contentWindow?.postMessage({ type: 'ai-hub-response', id, data: binary }, event.origin, [binary.content])
return
}
let payload = data
if (file) {
if (!(file.content instanceof ArrayBuffer) || file.content.byteLength > 20 * 1024 * 1024 || typeof file.name !== 'string') throw new Error('文件无效')

View File

@ -0,0 +1,59 @@
<template>
<div v-loading="loading">
<el-radio-group v-model="mode" class="mb-16">
<el-radio-button value="PATH">服务器目录</el-radio-button>
<el-radio-button value="UPLOAD">上传ZIP</el-radio-button>
</el-radio-group>
<template v-if="mode === 'PATH'">
<el-input v-model="path" placeholder="服务器模型绝对目录" class="mb-12" />
</template>
<el-upload v-else :auto-upload="false" :limit="1" accept=".zip" v-model:file-list="files" class="mb-12" :on-exceed="() => MsgError('请先移除已选文件')">
<el-button>选择模型包</el-button>
<template #tip><div class="el-upload__tip">ZIP ≤ 2GB;大模型使用服务器目录</div></template>
</el-upload>
<el-button type="primary" @click="importFiles" :disabled="mode === 'PATH' ? !path.trim() : !files.length">导入</el-button>
<div class="color-secondary mt-12 mb-12">文件导入不启动推理服务</div>
<el-table :data="records" max-height="300">
<el-table-column prop="name" label="名称" min-width="140" show-overflow-tooltip />
<el-table-column prop="format" label="格式" width="80" />
<el-table-column label="大小" width="85"><template #default="{ row }">{{ (row.size_bytes / 1024 ** 2).toFixed(1) }} MB</template></el-table-column>
<el-table-column label="操作" width="110"><template #default="{ row }">
<el-button link type="primary" @click="selected = row">详情</el-button>
<el-button v-if="row.format === 'HF'" link type="primary" @click="$emit('use', row)">使用</el-button>
</template></el-table-column>
</el-table>
<el-descriptions v-if="selected" :column="1" :label-width="80" border size="small" class="mt-16 import-details">
<el-descriptions-item label="目录">{{ selected.path }}</el-descriptions-item>
<el-descriptions-item label="文件">{{ selected.file_count }} 个,权重 {{ selected.weight_count }} 个</el-descriptions-item>
<el-descriptions-item label="元信息">{{ JSON.stringify(selected.metadata) }}</el-descriptions-item>
<el-descriptions-item label="文件检查">{{ selected.checks.join(';') }}</el-descriptions-item>
<el-descriptions-item label="运行状态">未验证</el-descriptions-item>
</el-descriptions>
</div>
</template>
<script setup lang="ts">
import { onMounted, ref } from 'vue'
import type { UploadUserFile } from 'element-plus'
import { get, post } from '@/request'
import { MsgError, MsgSuccess } from '@/utils/message'
defineEmits(['use'])
const mode = ref('PATH'), path = ref(''), files = ref<UploadUserFile[]>([])
const loading = ref(false), records = ref<any[]>([]), selected = ref<any>()
async function load() { records.value = (await get('/local_model/imports', undefined, loading)).data }
async function importFiles() {
let body: any = { path: path.value.trim() }
if (mode.value === 'UPLOAD') {
const file = files.value[0]?.raw
if (!file || !file.name.toLowerCase().endsWith('.zip') || file.size > 2 * 1024 ** 3) { MsgError('请选择2GB以内的ZIP模型包'); return }
body = new FormData(); body.append('file', file)
}
try {
selected.value = (await post('/local_model/imports', body, undefined, loading)).data
MsgSuccess('已导入'); path.value = ''; files.value = []; await load()
} catch { /* request interceptor displays the server error; retain the form */ }
}
onMounted(load)
</script>
<style scoped>
.import-details :deep(.el-descriptions__content) { overflow-wrap: anywhere; word-break: break-word; }
</style>

View File

@ -0,0 +1,89 @@
<template>
<el-dialog v-model="visible" title="模型载入" width="560px" :close-on-click-modal="false" @closed="stopPolling">
<el-form label-position="top" v-loading="saving">
<el-form-item label="模型"><el-input :model-value="model.name" readonly /></el-form-item>
<el-form-item label="加载方式">
<el-select v-model="form.strategy" class="w-full">
<el-option label="按需加载" value="ON_DEMAND" />
<el-option label="启动预加载" value="PRELOAD" />
</el-select>
</el-form-item>
<el-form-item label="加载优先级(数值越大越优先)">
<el-input-number v-model="form.priority" :min="0" :max="100" :precision="0" controls-position="right" />
</el-form-item>
<div class="flex-between mb-12">
<span>运行状态 <el-tag :type="state.status === 'READY' ? 'success' : state.status === 'FAILED' ? 'danger' : 'info'" class="ml-8">{{ statusLabel }}</el-tag></span>
<el-button link type="primary" :loading="checking" @click="refresh">刷新</el-button>
</div>
<div v-if="state.device" class="color-secondary mb-8">
设备:{{ state.device }}
<span v-if="state.gpu_bytes !== undefined"> · 显存:{{ (state.gpu_bytes / 1024 / 1024).toFixed(0) }} MB</span>
<span v-if="state.process_gpu_bytes !== undefined"> · 服务进程显存:{{ (state.process_gpu_bytes / 1024 / 1024).toFixed(0) }} MB</span>
</div>
<div v-if="state.loaded_at" class="color-secondary mb-8">加载时间:{{ new Date(state.loaded_at * 1000).toLocaleString() }}</div>
<div v-if="state.active_requests" class="color-secondary mb-8">当前请求:{{ state.active_requests }}</div>
<div v-if="state.error" class="color-danger mb-12">{{ state.error }}</div>
<el-button :disabled="dirty || busy || state.status === 'READY' || !ready" :loading="acting === 'load'" @click="run('load')">立即加载</el-button>
<el-button :disabled="dirty || busy || state.status !== 'READY' || !!state.active_requests" :loading="acting === 'unload'" @click="run('unload')">卸载</el-button>
</el-form>
<template #footer>
<el-button @click="visible = false">关闭</el-button>
<el-button type="primary" :loading="saving" :disabled="!ready || !!acting" @click="save">保存</el-button>
</template>
</el-dialog>
</template>
<script setup lang="ts">
import { ref, reactive, computed, onBeforeUnmount } from 'vue'
import { get, put, post } from '@/request'
import useStore from '@/stores'
import { MsgSuccess, MsgError } from '@/utils/message'
const { user } = useStore()
const emit = defineEmits(['change'])
const visible = ref(false), ready = ref(false), checking = ref(false), saving = ref(false)
const acting = ref(''), model = ref<any>({}), state = ref<any>({})
const form = reactive({ strategy: 'ON_DEMAND', priority: 50, idle_seconds: 300 })
const saved = ref('')
const dirty = computed(() => JSON.stringify(form) !== saved.value)
const busy = computed(() => !!acting.value || saving.value || ['LOADING', 'QUEUED'].includes(state.value.status))
const labels: Record<string, string> = { UNLOADED: '未加载', QUEUED: '等待加载', LOADING: '加载中', READY: '已加载', FAILED: '加载失败', UNAVAILABLE: '服务不可用', EXTERNAL: '外部服务' }
const statusLabel = computed(() => labels[state.value.status] || '读取中')
let timer: ReturnType<typeof setInterval> | undefined
const path = () => `/workspace/${model.value.workspace_id || user.getWorkspaceId()}/model/${model.value.id}/runtime`
function stopPolling() { if (timer) clearInterval(timer); timer = undefined }
async function refresh() {
if (checking.value || !visible.value) return
const id = model.value.id
const response: any = await get(path(), undefined, checking)
if (visible.value && model.value.id === id) state.value = response.data
}
async function open(value: any) {
stopPolling(); model.value = value; state.value = {}; ready.value = false; visible.value = true
const id = value.id
const response: any = await get(path(), undefined, checking)
if (!visible.value || model.value.id !== id) return
state.value = response.data
Object.assign(form, { strategy: response.data.strategy, priority: response.data.priority, idle_seconds: response.data.idle_seconds })
saved.value = JSON.stringify(form); ready.value = response.data.supported
timer = setInterval(() => { refresh().catch(() => {}) }, 5000)
}
async function save() {
if (!Number.isInteger(form.priority) || form.priority < 0 || form.priority > 100) { MsgError('加载优先级须为0~100的整数'); return }
const response: any = await put(path(), { ...form }, undefined, saving)
state.value = response.data; saved.value = JSON.stringify(form)
emit('change'); MsgSuccess('保存成功')
}
async function run(action: string) {
if (busy.value || dirty.value) return
acting.value = action
try {
const response: any = await post(path(), { action })
state.value = response.data
MsgSuccess(action === 'load' ? '加载请求已提交' : '已卸载')
} finally { acting.value = '' }
}
onBeforeUnmount(stopPolling)
defineExpose({ open })
</script>

View File

@ -0,0 +1,18 @@
<template>
<div class="mindmap-management p-16-24">
<h2 class="mb-16">思维导图管理</h2>
<el-card :body-style="{ padding: '0' }">
<HubManager @open-artifact="openArtifact" />
</el-card>
</div>
</template>
<script setup lang="ts">
import { useRouter } from 'vue-router'
import HubManager from '@/views/mindmap/HubManager.vue'
const router = useRouter()
function openArtifact(artifact: { id: string }) {
router.push({ path: '/mindmap', query: { artifact: artifact.id } })
}
</script>

View File

@ -0,0 +1,188 @@
<template>
<el-drawer v-model="visible" size="min(1180px, 100vw)" title="待激活审核" :destroy-on-close="false">
<div class="audit-toolbar">
<el-input
v-model="keyword"
placeholder="搜索用户名 / 姓名 / 邮箱 / 手机号"
clearable
style="width: 280px"
@keyup.enter="search"
/>
<el-select v-model="status" style="width: 120px" @change="search">
<el-option label="全部" value="UNAPPROVED" />
<el-option label="待审核" value="PENDING" />
<el-option label="已拒绝" value="REJECTED" />
</el-select>
<el-button type="primary" @click="search">查询</el-button>
<el-button @click="getList">刷新</el-button>
<span class="audit-count">共 <b>{{ total }}</b> 人</span>
</div>
<el-table :data="records" v-loading="loading" height="calc(100vh - 235px)">
<el-table-column prop="nick_name" label="昵称" width="90" show-overflow-tooltip />
<el-table-column prop="username" label="用户名" width="110" show-overflow-tooltip />
<el-table-column prop="email" label="注册邮箱" min-width="180" show-overflow-tooltip />
<el-table-column prop="phone" label="手机号" width="130">
<template #default="{ row }">{{ row.phone || '-' }}</template>
</el-table-column>
<el-table-column label="邮箱验证" width="90" align="center">
<template #default="{ row }">
<el-tag v-if="row.email_verified" type="success" size="small">已验证</el-tag>
<el-tag v-else type="info" size="small">未验证</el-tag>
</template>
</el-table-column>
<el-table-column prop="applied_at" label="申请时间" width="160" />
<el-table-column label="状态" width="90">
<template #default="{ row }">
<el-tag v-if="row.status === 'REJECTED'" type="danger" size="small">已拒绝</el-tag>
<el-tag v-else type="warning" size="small">待审核</el-tag>
</template>
</el-table-column>
<el-table-column prop="reject_reason" label="拒绝原因" min-width="150" show-overflow-tooltip>
<template #default="{ row }">{{ row.reject_reason || '-' }}</template>
</el-table-column>
<el-table-column label="操作" width="120" fixed="right">
<template #default="{ row }">
<template v-if="row.status === 'PENDING'">
<el-button type="primary" text @click="approve(row)">通过</el-button>
<el-button type="danger" text @click="reject(row)">拒绝</el-button>
</template>
<span v-else>-</span>
</template>
</el-table-column>
<template #empty>
<el-empty description="暂无记录" />
</template>
</el-table>
<div class="audit-pagination">
<el-pagination
v-model:current-page="pagination.current_page"
v-model:page-size="pagination.page_size"
:page-sizes="[10, 20, 50]"
:total="total"
layout="total, prev, pager, next, sizes"
@size-change="getList"
@current-change="getList"
/>
</div>
</el-drawer>
</template>
<script setup lang="ts">
import { reactive, ref } from 'vue'
import { ElMessageBox } from 'element-plus'
import userManageApi from '@/api/system/user-manage'
import { MsgSuccess } from '@/utils/message'
import { t } from '@/locales'
const emit = defineEmits(['refresh'])
const visible = ref(false)
const loading = ref(false)
const keyword = ref('')
const status = ref('UNAPPROVED')
const records = ref<any[]>([])
const total = ref(0)
const pagination = reactive({ current_page: 1, page_size: 10 })
function open() {
visible.value = true
pagination.current_page = 1
keyword.value = ''
status.value = 'UNAPPROVED'
getList()
}
function search() {
pagination.current_page = 1
getList()
}
function getList() {
loading.value = true
userManageApi
.getActivationPending(
{
current_page: pagination.current_page,
page_size: pagination.page_size,
keyword: keyword.value,
status: status.value,
},
loading,
)
.then((res: any) => {
records.value = res.data?.records || []
total.value = res.data?.total || 0
})
.finally(() => {
loading.value = false
})
}
function approve(row: any) {
ElMessageBox.confirm(
`确认激活「${row.nick_name || row.username}」?`,
'审核通过',
{
confirmButtonText: '确认通过',
cancelButtonText: t('common.cancel'),
type: 'warning',
},
)
.then(() => {
userManageApi
.approveActivation(row.id, { role_id: 'USER', workspace_id: 'default' }, loading)
.then(() => {
MsgSuccess('已激活')
getList()
emit('refresh')
})
})
.catch(() => {})
}
function reject(row: any) {
ElMessageBox.prompt('拒绝原因', '审核拒绝', {
confirmButtonText: '确认拒绝',
cancelButtonText: t('common.cancel'),
inputPlaceholder: '请输入拒绝原因',
inputValidator: (value: string) => (value && value.trim() ? true : '请填写拒绝原因'),
})
.then(({ value }: any) => {
userManageApi
.rejectActivation(row.id, { reason: value }, loading)
.then(() => {
MsgSuccess('已拒绝')
getList()
emit('refresh')
})
})
.catch(() => {})
}
defineExpose({ open })
</script>
<style scoped>
.audit-toolbar {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 14px;
}
.audit-count {
margin-left: auto;
color: #646a73;
font-size: 13px;
}
.audit-count b {
color: #3370ff;
font-size: 15px;
margin: 0 2px;
}
.audit-pagination {
display: flex;
justify-content: flex-end;
margin-top: 12px;
}
</style>

View File

@ -0,0 +1,161 @@
<template>
<el-dialog v-model="visible" title="分配角色" width="580px" :close-on-click-modal="false">
<div class="role-user">
<div class="role-user__avatar">{{ (detail.user?.nick_name || detail.user?.username || '?').slice(0, 1) }}</div>
<div class="role-user__info">
<div class="role-user__name">
{{ detail.user?.nick_name || '-' }}
<el-tag v-if="detail.user?.is_active" type="success" size="small">已启用</el-tag>
<el-tag v-else type="info" size="small">未启用</el-tag>
</div>
<div class="role-user__meta">{{ detail.user?.username }} · {{ detail.user?.email || '未填写邮箱' }}</div>
</div>
</div>
<el-form label-position="top" class="role-form">
<el-form-item label="角色">
<el-select v-model="form.role_id" placeholder="请选择角色" style="width: 100%">
<el-option
v-for="item in availableRoles"
:key="item.id"
:label="item.name"
:value="item.id"
/>
</el-select>
</el-form-item>
</el-form>
<div class="role-current">
<div class="role-current__title">当前授权</div>
<el-table :data="detail.assignments" size="small" border>
<el-table-column label="角色" min-width="120">
<template #default="{ row }">{{ roleName(row.role_id) }}</template>
</el-table-column>
<el-table-column prop="granted_at" label="授权时间" min-width="160">
<template #default="{ row }">{{ row.granted_at || '-' }}</template>
</el-table-column>
</el-table>
</div>
<template #footer>
<el-button @click="visible = false">{{ $t('common.cancel') }}</el-button>
<el-button type="primary" :loading="loading" :disabled="!form.role_id" @click="save">保存</el-button>
</template>
</el-dialog>
</template>
<script setup lang="ts">
import { reactive, ref } from 'vue'
import userManageApi from '@/api/system/user-manage'
import { MsgSuccess } from '@/utils/message'
const emit = defineEmits(['refresh'])
const visible = ref(false)
const loading = ref(false)
const detail = reactive<any>({ user: {}, assignments: [] })
const form = reactive({ role_id: '' })
const availableRoles = [
{ id: 'USER', name: '普通用户' },
{ id: 'ADMIN', name: '管理员' },
]
const roleMap: Record<string, string> = {
ADMIN: '管理员',
USER: '普通用户',
}
function roleName(id: string) {
return roleMap[id] || id
}
function open(row: any) {
visible.value = true
loading.value = true
detail.user = {}
detail.assignments = []
form.role_id = ''
userManageApi
.getUserRoles(row.id, loading)
.then((res: any) => {
const data = res.data || {}
detail.user = data.user || row
detail.assignments = (data.assignments || []).filter((item: any) => item.role_id in roleMap)
const first = detail.assignments?.[0]
const currentRole = data.user?.role || first?.role_id
form.role_id = currentRole in roleMap ? currentRole : ''
})
.finally(() => {
loading.value = false
})
}
function save() {
if (!detail.user?.id || !form.role_id || loading.value) return
loading.value = true
userManageApi
.putUserRoles(
detail.user.id,
{
role_settings: [
{
role_id: form.role_id,
},
],
},
loading,
)
.then(() => {
MsgSuccess('角色已更新')
visible.value = false
emit('refresh')
})
.finally(() => {
loading.value = false
})
}
defineExpose({ open })
</script>
<style scoped>
.role-user {
display: flex;
align-items: center;
gap: 12px;
padding: 4px 0 12px;
}
.role-user__avatar {
width: 42px;
height: 42px;
border-radius: 8px;
background: #e6f0ff;
color: #3370ff;
display: flex;
align-items: center;
justify-content: center;
font-size: 18px;
font-weight: 700;
}
.role-user__name {
font-size: 15px;
font-weight: 600;
color: #1f2329;
display: flex;
align-items: center;
gap: 8px;
}
.role-user__meta {
color: #646a73;
font-size: 12.5px;
margin-top: 3px;
}
.role-form {
margin-bottom: 4px;
}
.role-current__title {
font-size: 13px;
font-weight: 600;
color: #1f2329;
margin: 10px 0 8px;
}
</style>

View File

@ -0,0 +1,69 @@
<template>
<div class="parser-manager" v-loading="loading">
<div class="flex-between mb-16">
<h4>文档解析</h4>
<div><el-button @click="load">刷新</el-button><el-button v-if="resources.is_admin" type="primary" @click="create">新增解析服务</el-button></div>
</div>
<el-table :data="rows" row-key="id">
<el-table-column prop="name" label="名称" min-width="180" />
<el-table-column label="接口类型" width="160"><template #default="{ row }">{{ row.options?.protocol === 'v1' ? 'MinerU v1' : 'MinerU 原版' }}</template></el-table-column>
<el-table-column label="状态" width="100"><template #default="{ row }">{{ row.enabled ? '启用' : '停用' }}</template></el-table-column>
<el-table-column prop="updated_at" label="更新时间" min-width="190"><template #default="{ row }">{{ new Date(row.updated_at).toLocaleString() }}</template></el-table-column>
<el-table-column v-if="resources.is_admin" label="操作" min-width="230"><template #default="{ row }">
<el-button link type="primary" @click="open(row)">编辑</el-button>
<el-button link type="primary" :loading="testing === row.id" :disabled="!!testing && testing !== row.id" @click="test(row)">连接测试</el-button>
<el-button link type="primary" @click="toggle(row)">{{ row.enabled ? '停用' : '启用' }}</el-button>
</template></el-table-column>
<template #empty>暂无解析服务</template>
</el-table>
<el-dialog v-model="dialog" :title="form.id ? '编辑解析服务' : '新增解析服务'" width="640px" destroy-on-close :close-on-click-modal="false">
<el-form ref="formRef" :model="form" label-position="top">
<el-form-item label="名称" prop="name" :rules="[{ required: true, message: '请输入名称', whitespace: true }]"><el-input v-model="form.name" maxlength="128" /></el-form-item>
<el-form-item label="接口地址" prop="endpoint" :rules="[{ required: true, message: '请输入接口地址', whitespace: true }]"><el-input v-model="form.endpoint" placeholder="http://服务器:8100/v1" /></el-form-item>
<el-form-item label="API Key"><el-input v-model="form.api_key" type="password" show-password :placeholder="form.credential_configured ? '留空保留原密钥' : ''" autocomplete="new-password" /></el-form-item>
<el-form-item label="接口类型"><el-select v-model="form.options.protocol"><el-option label="MinerU v1 API" value="v1" /><el-option label="MinerU 原版API" value="legacy" /></el-select></el-form-item>
<el-form-item v-if="form.options.protocol === 'v1'" label="解析档位"><el-select v-model="form.options.tier"><el-option v-for="tier in ['flash', 'basic', 'standard', 'advanced']" :key="tier" :label="tier" :value="tier" /></el-select></el-form-item>
<template v-else>
<el-form-item label="解析后端"><el-select v-model="form.options.backend"><el-option label="传统管道" value="pipeline" /><el-option label="混合引擎" value="hybrid-auto-engine" /><el-option label="视觉引擎" value="vlm-auto-engine" /></el-select></el-form-item>
<el-form-item label="识别语言"><el-select v-model="form.options.language"><el-option v-for="item in languages" :key="item.value" v-bind="item" /></el-select></el-form-item>
</template>
<div class="mb-16"><el-checkbox v-model="form.options.ocr">强制OCR</el-checkbox><template v-if="form.options.protocol !== 'v1'"><el-checkbox v-model="form.options.table">表格</el-checkbox><el-checkbox v-model="form.options.formula">公式</el-checkbox></template></div>
<el-form-item label="授权用户"><el-select v-model="form.allowed_users" multiple filterable><el-option v-for="u in resources.users || []" :key="u.id" :label="u.name" :value="u.id" /></el-select></el-form-item>
<el-form-item label="启用"><el-switch v-model="form.enabled" /></el-form-item>
</el-form>
<template #footer><el-button @click="dialog = false">取消</el-button><el-button type="primary" :loading="saving" @click="save">保存</el-button></template>
</el-dialog>
</div>
</template>
<script setup lang="ts">
import { onMounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import type { FormInstance } from 'element-plus'
import { hubGet, hubPost, hubBase } from '@/api/ai-hub'
import { post } from '@/request'
const loading = ref(false), saving = ref(false), dialog = ref(false), testing = ref('')
const resources = ref<any>({ is_admin: false, users: [] }), rows = ref<any[]>([]), form = ref<any>({ options: {} })
const formRef = ref<FormInstance>()
const languages = [{ value: 'ch', label: '中文' }, { value: 'en', label: '英文' }, { value: 'japan', label: '日文' }, { value: 'korean', label: '韩文' }, { value: 'chinese_cht', label: '繁体中文' }]
async function load() {
loading.value = true
try { const [r, p] = await Promise.all([hubGet('resources'), hubGet('parsers')]); resources.value = r.data; rows.value = p.data }
finally { loading.value = false }
}
function create() { form.value = { name: '', endpoint: '', api_key: '', enabled: true, allowed_users: [], options: { protocol: 'v1', tier: 'basic', backend: 'pipeline', language: 'ch', table: true, formula: true, ocr: false } }; dialog.value = true }
async function open(row: any) { form.value = (await hubGet('parsers', row.id)).data; form.value.api_key = ''; dialog.value = true }
async function save() {
if (!await formRef.value?.validate().catch(() => false)) return
saving.value = true
try { const body = { ...form.value }; if (!body.api_key) delete body.api_key; await hubPost('parsers', body, body.id); dialog.value = false; ElMessage.success('已保存'); await load() }
finally { saving.value = false }
}
async function toggle(row: any) { loading.value = true; try { await hubPost('parsers', { enabled: !row.enabled }, row.id); await load() } finally { loading.value = false } }
async function test(row: any) {
testing.value = row.id
try { const r = await post(`${hubBase()}/parsers/${row.id}/test`, {}); ElMessage.success(`连接正常(${r.data.elapsed_ms} ms)`) }
finally { testing.value = '' }
}
onMounted(load)
</script>
<style scoped>.parser-manager { padding: 16px 24px; flex: 1; min-height: 0; overflow: auto; } .el-select { width: 100%; }</style>