完善知识库、模型管理和思维导图功能
parent
6df950ec3f
commit
69eb1eb681
|
|
@ -0,0 +1,202 @@
|
|||
"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
|
||||
import contextlib
|
||||
import itertools
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import Future
|
||||
|
||||
|
||||
def loading_policy(value=None):
|
||||
value = {} if value is None else value
|
||||
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
|
||||
raise ValueError('加载策略无效')
|
||||
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
|
||||
'idle_seconds': value.get('idle_seconds', 300)}
|
||||
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
|
||||
raise ValueError('加载方式无效')
|
||||
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
|
||||
raise ValueError('加载优先级须为0~100的整数')
|
||||
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
|
||||
raise ValueError('空闲时间须为30~86400秒')
|
||||
return result
|
||||
|
||||
|
||||
class ModelRuntime:
|
||||
def __init__(self, specs, adapter, clock=time.monotonic):
|
||||
self.specs, self.adapter, self.clock = specs, adapter, clock
|
||||
self.entries = {}
|
||||
self.lock = threading.RLock()
|
||||
self.queue = queue.PriorityQueue()
|
||||
self.sequence = itertools.count()
|
||||
self.load_sequence = itertools.count(1)
|
||||
self.worker = None
|
||||
|
||||
def _entry(self, model_id):
|
||||
model_id = str(model_id)
|
||||
spec = self.specs(model_id)
|
||||
spec['policy'] = loading_policy(spec.get('policy'))
|
||||
entry = self.entries.get(model_id)
|
||||
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
|
||||
if entry['leases'] or entry['state'] == 'LOADING':
|
||||
raise ValueError('模型配置已变更,请等待当前请求结束')
|
||||
self._discard(entry)
|
||||
entry = None
|
||||
if entry is None:
|
||||
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
|
||||
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
|
||||
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
|
||||
'load_order': None}
|
||||
self.entries[model_id] = entry
|
||||
previous = entry['spec']['policy']
|
||||
entry['spec'] = spec
|
||||
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
|
||||
entry['generation'] += 1
|
||||
self._put(entry)
|
||||
return entry
|
||||
|
||||
def _put(self, entry):
|
||||
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
|
||||
|
||||
def start(self):
|
||||
with self.lock:
|
||||
if self.worker is None:
|
||||
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
|
||||
self.worker.start()
|
||||
|
||||
def submit(self, model_id, start=True):
|
||||
with self.lock:
|
||||
entry = self._entry(model_id)
|
||||
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
|
||||
entry.update(state='UNLOADED', value=None, info={})
|
||||
if entry['state'] == 'READY':
|
||||
return entry
|
||||
if entry['state'] not in ('LOADING', 'QUEUED'):
|
||||
entry.update(state='QUEUED', error='', future=Future())
|
||||
entry['generation'] += 1
|
||||
self._put(entry)
|
||||
if start:
|
||||
self.start()
|
||||
return entry
|
||||
|
||||
def preload(self, model_ids):
|
||||
# Enqueue the complete startup batch before choosing the highest-priority model.
|
||||
with self.lock:
|
||||
for model_id in model_ids:
|
||||
self.submit(model_id, start=False)
|
||||
self.start()
|
||||
|
||||
def _work(self):
|
||||
while True:
|
||||
_, order, model_id, generation = self.queue.get()
|
||||
try:
|
||||
with self.lock:
|
||||
entry = self.entries.get(model_id)
|
||||
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
|
||||
continue
|
||||
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
|
||||
spec = entry['spec']
|
||||
future = entry['future']
|
||||
value, info = self.adapter.load(spec)
|
||||
with self.lock:
|
||||
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
|
||||
future.set_result(True)
|
||||
del value # The loader thread must not keep the last model alive after eviction.
|
||||
except Exception as exc:
|
||||
with self.lock:
|
||||
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
|
||||
if not entry['future'].done():
|
||||
entry['future'].set_exception(ValueError(entry['error']))
|
||||
finally:
|
||||
self.queue.task_done()
|
||||
|
||||
def acquire(self, model_id, remote=False, timeout=600):
|
||||
with self.lock:
|
||||
entry = self.submit(model_id)
|
||||
token = uuid.uuid4().hex
|
||||
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
|
||||
future = entry['future'] if entry['state'] != 'READY' else None
|
||||
try:
|
||||
if future:
|
||||
future.result(timeout=timeout)
|
||||
return token
|
||||
except Exception:
|
||||
self.release(model_id, token)
|
||||
raise
|
||||
|
||||
def release(self, model_id, token):
|
||||
with self.lock:
|
||||
entry = self.entries.get(str(model_id))
|
||||
if entry:
|
||||
entry['leases'].pop(token, None)
|
||||
entry['last_used'] = self.clock()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def inference(self, model_id):
|
||||
token = self.acquire(model_id)
|
||||
try:
|
||||
with self.lock:
|
||||
value = self.entries[str(model_id)]['value']
|
||||
yield value
|
||||
finally:
|
||||
self.release(model_id, token)
|
||||
|
||||
def _discard(self, entry):
|
||||
if entry['value'] is not None:
|
||||
value = entry['value']
|
||||
self.adapter.unload(entry['spec'], value)
|
||||
entry['value'] = None
|
||||
del value
|
||||
self.adapter.collect()
|
||||
entry.update(state='UNLOADED', loaded_at=None, info={})
|
||||
|
||||
def unload(self, model_id):
|
||||
with self.lock:
|
||||
entry = self._entry(model_id)
|
||||
self._expire_leases(entry)
|
||||
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
|
||||
raise ValueError('模型正在使用或加载,请稍后再试')
|
||||
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
|
||||
entry['value'] = True
|
||||
self._discard(entry)
|
||||
|
||||
def configure(self, model_id):
|
||||
with self.lock:
|
||||
entry = self._entry(model_id)
|
||||
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
|
||||
if warm_again:
|
||||
entry.update(state='UNLOADED', value=None)
|
||||
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
|
||||
self.submit(model_id)
|
||||
|
||||
def _expire_leases(self, entry):
|
||||
now = self.clock()
|
||||
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
|
||||
|
||||
def sweep(self):
|
||||
with self.lock:
|
||||
for entry in list(self.entries.values()):
|
||||
self._expire_leases(entry)
|
||||
policy = entry['spec']['policy']
|
||||
# Ollama owns its idle timer; do not unload models used by other clients.
|
||||
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
|
||||
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
|
||||
self.clock() - entry['last_used'] >= policy['idle_seconds']):
|
||||
self._discard(entry)
|
||||
|
||||
def status(self, model_id):
|
||||
with self.lock:
|
||||
entry = self._entry(model_id)
|
||||
self._expire_leases(entry)
|
||||
# Reconcile native residency, including expiration outside this process.
|
||||
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
|
||||
info = self.adapter.probe(entry['spec'])
|
||||
if info:
|
||||
entry.update(state='READY', info=info, value=True)
|
||||
elif entry['state'] == 'READY':
|
||||
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
|
||||
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
|
||||
'active_requests': len(entry['leases']), 'error': entry['error'],
|
||||
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
|
||||
'load_order': entry['load_order'], **entry['info']}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
"""Authenticated internal model-service RPC; never forwards user-supplied endpoints."""
|
||||
import contextlib
|
||||
import asyncio
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _secret():
|
||||
if os.environ.get('SERVER_NAME') == 'local_model':
|
||||
from local_model.models.system_setting import SystemSetting
|
||||
else:
|
||||
from system_manage.models import SystemSetting
|
||||
meta = SystemSetting.objects.filter(type=1).values_list('meta', flat=True).first()
|
||||
if not meta or not meta.get('value'):
|
||||
raise ValueError('模型服务认证未配置')
|
||||
return hashlib.sha256(('model-runtime-v1:' + meta['value']).encode()).digest()
|
||||
|
||||
|
||||
def signed_request(payload):
|
||||
data = {**payload, 'timestamp': time.time(), 'nonce': uuid.uuid4().hex}
|
||||
body = json.dumps(data, separators=(',', ':'), sort_keys=True).encode()
|
||||
signature = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
|
||||
return body, signature
|
||||
|
||||
|
||||
def verify_request(body, signature):
|
||||
expected = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
|
||||
if not signature or not hmac.compare_digest(expected, signature):
|
||||
raise ValueError('模型服务认证失败')
|
||||
data = json.loads(body)
|
||||
if abs(time.time() - float(data.get('timestamp', 0))) > 60:
|
||||
raise ValueError('模型服务请求已过期')
|
||||
return data
|
||||
|
||||
|
||||
def runtime_request(operation, model_id, **extra):
|
||||
from maxkb.const import CONFIG
|
||||
body, signature = signed_request({'operation': operation, 'model_id': str(model_id), **extra})
|
||||
host = CONFIG.get('LOCAL_MODEL_HOST')
|
||||
url = f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{host}:{CONFIG.get("LOCAL_MODEL_PORT")}{CONFIG.get_admin_path()}/api/model_runtime'
|
||||
try:
|
||||
response = requests.post(url, data=body, headers={'Content-Type': 'application/json',
|
||||
'X-Model-Runtime-Signature': signature},
|
||||
timeout=(5, 620 if operation == 'acquire' else 15))
|
||||
result = response.json()
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
raise ValueError('本地模型服务不可用') from exc
|
||||
if response.status_code != 200 or result.get('code') != 200:
|
||||
raise ValueError(result.get('message') or '模型服务请求失败')
|
||||
return result['data']
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def inference_lease(model_id):
|
||||
if not model_id:
|
||||
# Credential validation has no saved resource and must keep its original behavior.
|
||||
yield None
|
||||
return
|
||||
data = runtime_request('acquire', model_id)
|
||||
try:
|
||||
yield data
|
||||
finally:
|
||||
try:
|
||||
runtime_request('release', model_id, lease=data['lease'])
|
||||
except ValueError:
|
||||
pass # Server expires abandoned leases; never hide the inference result.
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def async_inference_lease(model_id):
|
||||
if not model_id:
|
||||
yield None
|
||||
return
|
||||
data = await asyncio.to_thread(runtime_request, 'acquire', model_id)
|
||||
try:
|
||||
yield data
|
||||
finally:
|
||||
try:
|
||||
await asyncio.to_thread(runtime_request, 'release', model_id, lease=data['lease'])
|
||||
except ValueError:
|
||||
pass
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""按完整表格行分段;表头重复,重叠行不跨表/工作表。"""
|
||||
def split_table_rows(header, rows, limit, overlap=0):
|
||||
limit, overlap = int(limit), int(overlap)
|
||||
if limit <= 0 or not 0 <= overlap < limit:
|
||||
raise ValueError("Invalid table splitting limits")
|
||||
chunks, current = [], []
|
||||
length = len(header)
|
||||
for row in rows:
|
||||
if current and length + len(row) > limit:
|
||||
chunks.append({"content": header + "".join(current), "title": ""})
|
||||
carry, carried = [], 0
|
||||
if overlap:
|
||||
for previous in reversed(current):
|
||||
carry.insert(0, previous)
|
||||
carried += len(previous)
|
||||
if carried >= overlap:
|
||||
break
|
||||
# 必须留出至少一条新行的空间,不能只重复旧行。
|
||||
while carry and len(header) + sum(map(len, carry)) + len(row) > limit:
|
||||
carry.pop(0)
|
||||
current = carry
|
||||
length = len(header) + sum(map(len, current))
|
||||
current.append(row)
|
||||
length += len(row)
|
||||
if current:
|
||||
chunks.append({"content": header + "".join(current), "title": ""})
|
||||
return chunks
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
|
||||
import math
|
||||
import unicodedata
|
||||
|
||||
from langchain_core.embeddings import Embeddings
|
||||
|
||||
|
||||
def validate_params(params):
|
||||
if (not isinstance(params, dict) or len(params) > 52 or
|
||||
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
|
||||
raise ValueError('可选参数最多50项')
|
||||
result, names = {}, set()
|
||||
for key, value in params.items():
|
||||
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
|
||||
raise ValueError('参数名称须为1~64个字符')
|
||||
key = key.strip()
|
||||
normalized = unicodedata.normalize('NFKC', key).casefold()
|
||||
if normalized in names:
|
||||
raise ValueError('参数名称不能重复')
|
||||
names.add(normalized)
|
||||
if value is not None and (not isinstance(value, (str, int, float, bool)) or
|
||||
isinstance(value, str) and len(value) > 2000 or
|
||||
isinstance(value, float) and not math.isfinite(value)):
|
||||
raise ValueError('参数值须为2000字以内的文本或数值')
|
||||
result[key] = value
|
||||
batch = result.get('batch_size')
|
||||
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
|
||||
raise ValueError('批处理数量须为1~256的整数')
|
||||
if result.get('normalize') is not None and type(result['normalize']) is not bool:
|
||||
raise ValueError('向量归一化参数无效')
|
||||
for key in ('query_prefix', 'document_prefix'):
|
||||
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
|
||||
raise ValueError('向量指令须为2000字以内的文本')
|
||||
return {key: value for key, value in result.items() if value is not None and
|
||||
(value != '' or key not in {'query_prefix', 'document_prefix'})}
|
||||
|
||||
|
||||
def knowledge_params(knowledge):
|
||||
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
|
||||
return {key: value for key, value in params.items()
|
||||
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
|
||||
|
||||
|
||||
class ConfiguredEmbedding(Embeddings):
|
||||
def __init__(self, model, params):
|
||||
self.model = model
|
||||
self.params = params
|
||||
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
|
||||
|
||||
def _vector(self, vector):
|
||||
if not self.params.get('normalize'):
|
||||
return vector
|
||||
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
|
||||
return [float(v) / norm for v in vector] if norm else vector
|
||||
|
||||
def embed_query(self, text):
|
||||
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
|
||||
|
||||
def embed_documents(self, texts):
|
||||
prefix = self.params.get('document_prefix', '')
|
||||
result = []
|
||||
for start in range(0, len(texts), self.batch_size):
|
||||
result.extend(self._vector(v) for v in self.model.embed_documents(
|
||||
[prefix + text for text in texts[start:start + self.batch_size]]))
|
||||
return result
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Generated for independent knowledge index rebuild task
|
||||
import uuid_utils.compat
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('knowledge', '0003_tag_documenttag'),
|
||||
('users', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='KnowledgeRebuildTask',
|
||||
fields=[
|
||||
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
|
||||
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
|
||||
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')),
|
||||
('status', models.CharField(choices=[('PENDING', '待执行'), ('RUNNING', '执行中'), ('SUCCESS', '成功'), ('PARTIAL', '部分成功'), ('FAILED', '失败')], db_index=True, default='PENDING', max_length=20, verbose_name='任务状态')),
|
||||
('config_snapshot', models.JSONField(default=dict, verbose_name='配置快照')),
|
||||
('document_total', models.IntegerField(default=0, verbose_name='文档总数')),
|
||||
('document_success', models.IntegerField(default=0, verbose_name='成功文档数')),
|
||||
('document_failed', models.IntegerField(default=0, verbose_name='失败文档数')),
|
||||
('error_message', models.TextField(blank=True, default='', verbose_name='失败原因')),
|
||||
('started_at', models.DateTimeField(blank=True, null=True, verbose_name='开始时间')),
|
||||
('finished_at', models.DateTimeField(blank=True, null=True, verbose_name='结束时间')),
|
||||
('celery_task_id', models.CharField(blank=True, default='', max_length=128, verbose_name='Celery任务id')),
|
||||
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='rebuild_tasks', to='knowledge.knowledge', verbose_name='知识库')),
|
||||
('operator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='操作人')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'knowledge_rebuild_task',
|
||||
'indexes': [models.Index(fields=['knowledge', 'status'], name='knowledge_r_knowled_0a6e2f_idx')],
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# Generated for rebuild task attempt polling
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('knowledge', '0004_knowledge_rebuild_task'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='knowledgerebuildtask',
|
||||
name='attempt',
|
||||
field=models.IntegerField(default=0, verbose_name='汇总轮询次数'),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
# coding=utf-8
|
||||
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
|
||||
from collections.abc import Mapping
|
||||
import re
|
||||
|
||||
from django.db import transaction
|
||||
from django.db.models import QuerySet
|
||||
from django.utils import timezone
|
||||
from rest_framework import serializers
|
||||
|
||||
from common.exception.app_exception import AppApiException
|
||||
from knowledge.models import Knowledge
|
||||
|
||||
METADATA_FIELDS = [
|
||||
{"field": "business_domain", "label": "业务领域"},
|
||||
{"field": "data_source", "label": "数据来源"},
|
||||
{"field": "owner_dept", "label": "责任部门"},
|
||||
{"field": "business_version", "label": "业务版本"},
|
||||
]
|
||||
METADATA_MAX_LEN = 64
|
||||
|
||||
|
||||
def _fmt(value):
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
value = timezone.localtime(value)
|
||||
except Exception:
|
||||
pass
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
|
||||
query = QuerySet(Knowledge)
|
||||
if lock:
|
||||
query = query.select_for_update()
|
||||
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
|
||||
if knowledge is None:
|
||||
raise AppApiException(500, "知识库不存在")
|
||||
return knowledge
|
||||
|
||||
|
||||
def _creator_candidates(workspace_id):
|
||||
from users.models import User, UserActivationApplication
|
||||
from common.database_model_manage.database_model_manage import DatabaseModelManage
|
||||
from django.db.models import Q
|
||||
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
|
||||
query = User.objects.filter(is_active=True).exclude(id__in=pending)
|
||||
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
|
||||
if membership is not None:
|
||||
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
|
||||
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
|
||||
return query
|
||||
|
||||
|
||||
def _metadata_fields(business):
|
||||
# An explicit empty list means all custom fields were removed; do not revive old fields.
|
||||
if isinstance(business.get('custom_fields'), list):
|
||||
return business['custom_fields']
|
||||
return [{'name': item['label'], 'value': business[item['field']]}
|
||||
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
|
||||
|
||||
|
||||
def _validate_custom_fields(items):
|
||||
import unicodedata
|
||||
if not isinstance(items, list) or len(items) > 50:
|
||||
raise AppApiException(500, '元数据最多50项')
|
||||
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
|
||||
result, names = [], set()
|
||||
for item in items:
|
||||
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
|
||||
raise AppApiException(500, '元数据需包含名称和值')
|
||||
name, value = item['name'], item['value']
|
||||
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
|
||||
raise AppApiException(500, '元数据名称须为1~64个字符')
|
||||
name = name.strip()
|
||||
normalized = unicodedata.normalize('NFKC', name).casefold()
|
||||
if normalized in reserved:
|
||||
raise AppApiException(500, '不能使用固定属性名称')
|
||||
if normalized in names:
|
||||
raise AppApiException(500, '元数据名称不能重复')
|
||||
if not isinstance(value, str) or len(value) > 2000:
|
||||
raise AppApiException(500, '元数据值须为2000字以内的文本')
|
||||
names.add(normalized)
|
||||
result.append({'name': name, 'value': value.strip()})
|
||||
return result
|
||||
|
||||
|
||||
class KnowledgeMetadataSerializer(serializers.Serializer):
|
||||
"""Fixed system attributes plus freely named business metadata; save atomically."""
|
||||
|
||||
@staticmethod
|
||||
def one(workspace_id: str, knowledge_id: str):
|
||||
from users.models import User
|
||||
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||||
data = (knowledge.meta or {}).get('business_metadata') or {}
|
||||
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
|
||||
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
|
||||
for u in _creator_candidates(workspace_id).order_by('nick_name')]
|
||||
if creator and not any(u['id'] == str(creator.id) for u in candidates):
|
||||
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
|
||||
return {
|
||||
'knowledge_id': str(knowledge.id),
|
||||
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
|
||||
'creator_name': (creator.nick_name or creator.username) if creator else '',
|
||||
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
|
||||
'creator_options': candidates,
|
||||
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
|
||||
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||||
if not isinstance(data, Mapping):
|
||||
raise AppApiException(500, '元数据必须为属性对象')
|
||||
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
|
||||
if set(data) - (set(legacy) | {'fields', 'user_id'}):
|
||||
raise AppApiException(500, '包含不支持的元数据属性')
|
||||
if 'fields' in data and set(data) & set(legacy):
|
||||
raise AppApiException(500, '不能混用新旧元数据格式')
|
||||
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
|
||||
values = {}
|
||||
for key, label in legacy.items():
|
||||
if key not in data:
|
||||
continue
|
||||
value = '' if data[key] is None else data[key]
|
||||
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
|
||||
raise AppApiException(500, f'{label}须为64字以内的文本')
|
||||
values[key] = value.strip()
|
||||
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||||
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
|
||||
try:
|
||||
creator_id = serializers.UUIDField().run_validation(data['user_id'])
|
||||
except serializers.ValidationError:
|
||||
raise AppApiException(500, '请选择有效创建者')
|
||||
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
|
||||
raise AppApiException(500, '创建者须为已激活且启用的成员')
|
||||
knowledge.user_id = creator_id
|
||||
meta = dict(knowledge.meta or {})
|
||||
business = dict(meta.get('business_metadata') or {})
|
||||
if fields is not None:
|
||||
business['custom_fields'] = fields
|
||||
elif 'custom_fields' in business:
|
||||
current = list(_metadata_fields(business))
|
||||
for key, value in values.items():
|
||||
existing = next((item for item in current if item['name'] == legacy[key]), None)
|
||||
if existing is not None:
|
||||
existing['value'] = value
|
||||
elif value:
|
||||
current.append({'name': legacy[key], 'value': value})
|
||||
business['custom_fields'] = _validate_custom_fields(current)
|
||||
business.update(values)
|
||||
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
|
||||
updated_at=_fmt(timezone.now()))
|
||||
meta['business_metadata'] = business
|
||||
knowledge.meta = meta
|
||||
knowledge.save(update_fields=['meta', 'user', 'update_time'])
|
||||
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
|
||||
|
||||
|
||||
class KnowledgeSegmentationSerializer(serializers.Serializer):
|
||||
"""知识库分段规则:查看 / 修改。
|
||||
|
||||
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
|
||||
保存后新建文档上传时默认使用该规则,上传时可覆盖。
|
||||
"""
|
||||
|
||||
DEFAULTS = {"chunk_size": 4096, "chunk_overlap": 0}
|
||||
|
||||
@staticmethod
|
||||
def one(workspace_id: str, knowledge_id: str):
|
||||
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||||
data = dict((knowledge.meta or {}).get('segmentation') or {})
|
||||
result = {
|
||||
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
|
||||
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
|
||||
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
|
||||
}
|
||||
result.update({
|
||||
"updated_by": data.get("updated_by_name") or "",
|
||||
"updated_at": data.get("updated_at") or "",
|
||||
"knowledge_id": str(knowledge.id),
|
||||
})
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||||
if not isinstance(data, Mapping):
|
||||
raise AppApiException(500, "分段规则必须为属性对象")
|
||||
def integer(value):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||||
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
|
||||
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||||
return int(value)
|
||||
chunk_size = integer(data.get("chunk_size"))
|
||||
chunk_overlap = integer(data.get("chunk_overlap", 0))
|
||||
if chunk_size < 50 or chunk_size > 100000:
|
||||
raise AppApiException(500, "分段长度取值范围为 50~100000")
|
||||
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
|
||||
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||||
meta = dict(knowledge.meta or {})
|
||||
meta['segmentation'] = {
|
||||
"chunk_size": chunk_size,
|
||||
"chunk_overlap": chunk_overlap,
|
||||
"updated_by": str(user.id),
|
||||
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
|
||||
"updated_at": _fmt(timezone.now()),
|
||||
}
|
||||
knowledge.meta = meta
|
||||
knowledge.save(update_fields=["meta", "update_time"])
|
||||
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
# coding=utf-8
|
||||
"""知识库独立索引重建任务序列化器。"""
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from rest_framework import serializers
|
||||
|
||||
from common.exception.app_exception import AppApiException
|
||||
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
|
||||
from knowledge.task.rebuild import rebuild_knowledge_index
|
||||
from models_provider.models import Model
|
||||
|
||||
|
||||
def _fmt(value):
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
value = timezone.localtime(value)
|
||||
except Exception:
|
||||
pass
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def serialize_task(task: KnowledgeRebuildTask):
|
||||
total = task.document_total or 0
|
||||
success = task.document_success or 0
|
||||
failed = task.document_failed or 0
|
||||
processed = success + failed
|
||||
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
|
||||
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
|
||||
return {
|
||||
"id": str(task.id),
|
||||
"knowledge_id": str(task.knowledge_id),
|
||||
"workspace_id": task.workspace_id,
|
||||
"status": task.status,
|
||||
"status_label": status_label,
|
||||
"config_snapshot": task.config_snapshot or {},
|
||||
"document_total": total,
|
||||
"document_success": success,
|
||||
"document_failed": failed,
|
||||
"progress": progress,
|
||||
"error_message": task.error_message or "",
|
||||
"created_at": _fmt(task.create_time),
|
||||
"started_at": _fmt(task.started_at),
|
||||
"finished_at": _fmt(task.finished_at),
|
||||
}
|
||||
|
||||
|
||||
class KnowledgeRebuildSerializer(serializers.Serializer):
|
||||
@staticmethod
|
||||
def create(workspace_id: str, knowledge_id: str, user_id: str):
|
||||
# 锁住知识库行,串行化同一知识库的检查和任务创建。
|
||||
with transaction.atomic():
|
||||
knowledge = Knowledge.objects.select_for_update().filter(
|
||||
id=knowledge_id, workspace_id=workspace_id
|
||||
).first()
|
||||
if knowledge is None:
|
||||
raise AppApiException(500, "知识库不存在")
|
||||
if KnowledgeRebuildTask.objects.filter(
|
||||
knowledge_id=knowledge_id,
|
||||
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
||||
).exists():
|
||||
raise AppApiException(500, "该知识库已有重建任务在执行")
|
||||
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
|
||||
if model is None:
|
||||
raise AppApiException(500, "知识库未配置可用的向量模型")
|
||||
document_ids = [str(pk) for pk in Document.objects.filter(
|
||||
knowledge_id=knowledge_id).values_list("id", flat=True)]
|
||||
if not document_ids:
|
||||
raise AppApiException(500, "知识库没有可重建的文档")
|
||||
snapshot = {
|
||||
"embedding_model_id": str(model.id),
|
||||
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
|
||||
"embedding_model_name": model.name,
|
||||
"embedding_model": model.model_name,
|
||||
"knowledge_name": knowledge.name,
|
||||
"document_ids": document_ids,
|
||||
"document_total": len(document_ids),
|
||||
"snapshot_at": _fmt(timezone.now()),
|
||||
}
|
||||
task = KnowledgeRebuildTask.objects.create(
|
||||
knowledge_id=knowledge_id, workspace_id=workspace_id,
|
||||
status=RebuildStatus.PENDING, config_snapshot=snapshot,
|
||||
document_total=len(document_ids), operator_id=user_id,
|
||||
)
|
||||
|
||||
def dispatch():
|
||||
try:
|
||||
result = rebuild_knowledge_index.delay(str(task.id))
|
||||
# 不覆盖 worker 已经更新的状态。
|
||||
KnowledgeRebuildTask.objects.filter(id=task.id).update(
|
||||
celery_task_id=getattr(result, "id", "") or "")
|
||||
except Exception as exc:
|
||||
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
|
||||
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
|
||||
finished_at=timezone.now(), update_time=timezone.now())
|
||||
|
||||
transaction.on_commit(dispatch)
|
||||
task.refresh_from_db()
|
||||
return serialize_task(task)
|
||||
|
||||
@staticmethod
|
||||
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
|
||||
query_set = KnowledgeRebuildTask.objects.filter(
|
||||
knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||
).order_by("-create_time")
|
||||
total = query_set.count()
|
||||
start = max(0, (current_page - 1) * page_size)
|
||||
records = [serialize_task(item) for item in query_set[start:start + page_size]]
|
||||
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
|
||||
|
||||
@staticmethod
|
||||
def one(workspace_id: str, knowledge_id: str, task_id: str):
|
||||
task = KnowledgeRebuildTask.objects.filter(
|
||||
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||
).first()
|
||||
if task is None:
|
||||
raise AppApiException(500, "重建任务不存在")
|
||||
return serialize_task(task)
|
||||
|
||||
@staticmethod
|
||||
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
|
||||
old_task = KnowledgeRebuildTask.objects.filter(
|
||||
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||
).first()
|
||||
if old_task is None:
|
||||
raise AppApiException(500, "重建任务不存在")
|
||||
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
|
||||
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
|
||||
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
# coding=utf-8
|
||||
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from common.db.sql_execute import sql_execute
|
||||
from common.event.listener_manage import ListenerManagement
|
||||
from common.utils.logger import maxkb_logger
|
||||
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
|
||||
TaskType, State, Status)
|
||||
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
|
||||
from ops import celery_app
|
||||
|
||||
POLL_SECONDS = 5
|
||||
MAX_POLLS = 120
|
||||
|
||||
|
||||
def _index_exists(knowledge_id: str) -> bool:
|
||||
rows = sql_execute(
|
||||
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
|
||||
[f"embedding_hnsw_idx_{knowledge_id}"]
|
||||
)
|
||||
return bool(rows)
|
||||
|
||||
|
||||
def _fail(task_id, message):
|
||||
KnowledgeRebuildTask.objects.filter(
|
||||
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
||||
).update(status=RebuildStatus.FAILED, error_message=message,
|
||||
finished_at=timezone.now(), update_time=timezone.now())
|
||||
|
||||
|
||||
@celery_app.task(name="celery:rebuild_knowledge_index")
|
||||
def rebuild_knowledge_index(task_id: str):
|
||||
# 重复投递不能重复清空向量或重复派发。
|
||||
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
|
||||
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
|
||||
attempt=0, update_time=timezone.now())
|
||||
if not claimed:
|
||||
return
|
||||
try:
|
||||
task = KnowledgeRebuildTask.objects.get(id=task_id)
|
||||
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||||
if knowledge is None:
|
||||
raise ValueError("知识库不存在")
|
||||
snapshot = task.config_snapshot or {}
|
||||
model_id = snapshot.get("embedding_model_id")
|
||||
if not model_id or str(knowledge.embedding_model_id) != model_id:
|
||||
raise ValueError("向量模型已变更,请重新发起任务")
|
||||
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
|
||||
raise ValueError('嵌入参数已变更,请重新发起任务')
|
||||
# 兼容历史任务;新任务在受理时即固定文档范围。
|
||||
document_ids = snapshot.get("document_ids")
|
||||
if document_ids is None:
|
||||
document_ids = [str(pk) for pk in Document.objects.filter(
|
||||
knowledge_id=knowledge.id).values_list("id", flat=True)]
|
||||
snapshot["document_ids"] = document_ids
|
||||
task.config_snapshot = snapshot
|
||||
task.save(update_fields=["config_snapshot", "update_time"])
|
||||
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
|
||||
if not documents or len(documents) != len(document_ids):
|
||||
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
|
||||
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
|
||||
raise ValueError("文档正在向量化,请完成后重新发起")
|
||||
|
||||
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
|
||||
drop_knowledge_index(knowledge_id=str(knowledge.id))
|
||||
task.document_total = len(document_ids)
|
||||
task.save(update_fields=["document_total", "update_time"])
|
||||
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
|
||||
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
|
||||
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
|
||||
ListenerManagement.update_status(
|
||||
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
|
||||
TaskType.EMBEDDING, State.PENDING)
|
||||
from knowledge.task.embedding import embedding_by_document
|
||||
dispatch_errors = []
|
||||
for document in documents:
|
||||
try:
|
||||
embedding_by_document.delay(str(document.id), model_id)
|
||||
except Exception as exc:
|
||||
ListenerManagement.update_status(
|
||||
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
|
||||
dispatch_errors.append(f"{document.name}:{exc}")
|
||||
if dispatch_errors:
|
||||
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
|
||||
task.save(update_fields=["error_message", "update_time"])
|
||||
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
|
||||
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
|
||||
except Exception as exc:
|
||||
_fail(task_id, f"重建任务启动失败:{exc}")
|
||||
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
|
||||
|
||||
|
||||
@celery_app.task(name="celery:finalize_rebuild_task")
|
||||
def finalize_rebuild_task(task_id: str):
|
||||
try:
|
||||
with transaction.atomic():
|
||||
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
|
||||
if task is None or task.status != RebuildStatus.RUNNING:
|
||||
return
|
||||
snapshot = task.config_snapshot or {}
|
||||
document_ids = snapshot.get("document_ids", [])
|
||||
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
|
||||
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
|
||||
success = states.count(State.SUCCESS)
|
||||
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
|
||||
for state in states) + len(document_ids) - len(documents)
|
||||
pending = len(document_ids) - success - failed
|
||||
task.document_total = len(document_ids)
|
||||
task.document_success = success
|
||||
task.document_failed = failed
|
||||
if pending:
|
||||
task.attempt += 1
|
||||
if task.attempt < MAX_POLLS:
|
||||
task.save(update_fields=["document_total", "document_success", "document_failed",
|
||||
"attempt", "update_time"])
|
||||
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
|
||||
(task_id,), countdown=POLL_SECONDS))
|
||||
return
|
||||
task.status = RebuildStatus.FAILED
|
||||
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
|
||||
else:
|
||||
knowledge_id = str(task.knowledge_id)
|
||||
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||||
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
|
||||
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
|
||||
raise ValueError("向量模型已变更,请重新发起任务")
|
||||
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
|
||||
create_knowledge_index(knowledge_id=knowledge_id)
|
||||
index_ready = _index_exists(knowledge_id)
|
||||
if success and failed == 0 and index_ready:
|
||||
task.status = RebuildStatus.SUCCESS
|
||||
task.error_message = ""
|
||||
elif success and failed and index_ready:
|
||||
task.status = RebuildStatus.PARTIAL
|
||||
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
|
||||
else:
|
||||
task.status = RebuildStatus.FAILED
|
||||
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
|
||||
task.finished_at = timezone.now()
|
||||
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
|
||||
"error_message", "finished_at", "attempt", "update_time"])
|
||||
except Exception as exc:
|
||||
_fail(task_id, f"重建结果汇总失败:{exc}")
|
||||
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""Celery 任务发现入口,确保独立重建和向量化任务被 Worker 注册。"""
|
||||
from knowledge.task import embedding # noqa: F401
|
||||
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task # noqa: F401
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from knowledge.embedding_params import ConfiguredEmbedding, validate_params, knowledge_params
|
||||
|
||||
|
||||
class RecordingModel:
|
||||
batch_size = 10
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def embed_query(self, text):
|
||||
self.calls.append(('query', text))
|
||||
return [3., 4.]
|
||||
|
||||
def embed_documents(self, texts):
|
||||
self.calls.append(('documents', texts))
|
||||
return [[3., 4.] for _ in texts]
|
||||
|
||||
|
||||
class EmbeddingParamsTests(unittest.TestCase):
|
||||
def test_inference_uses_separate_instructions_and_same_normalization(self):
|
||||
base = RecordingModel()
|
||||
model = ConfiguredEmbedding(base, {'normalize': True, 'query_prefix': '检索:', 'document_prefix': '正文:'})
|
||||
self.assertEqual(model.embed_query('内容'), [.6, .8])
|
||||
self.assertEqual(model.embed_documents(['内容']), [[.6, .8]])
|
||||
self.assertEqual(base.calls, [('query', '检索:内容'), ('documents', ['正文:内容'])])
|
||||
|
||||
def test_batch_size_is_applied_to_real_calls(self):
|
||||
base = RecordingModel()
|
||||
model = ConfiguredEmbedding(base, {'batch_size': 2})
|
||||
self.assertEqual(len(model.embed_documents(['a', 'b', 'c', 'd', 'e'])), 5)
|
||||
self.assertEqual([len(call[1]) for call in base.calls], [2, 2, 1])
|
||||
|
||||
def test_knowledge_overrides_do_not_leak_between_shared_models(self):
|
||||
base = RecordingModel()
|
||||
first = ConfiguredEmbedding(base, {'query_prefix': 'A'})
|
||||
second = ConfiguredEmbedding(base, {'query_prefix': 'B'})
|
||||
first.embed_query('x'); second.embed_query('x'); first.embed_query('x')
|
||||
self.assertEqual(base.calls, [('query', 'Ax'), ('query', 'Bx'), ('query', 'Ax')])
|
||||
self.assertEqual(base.batch_size, 10)
|
||||
|
||||
def test_invalid_inputs_are_rejected(self):
|
||||
for params in ([], {'batch_size': True}, {'batch_size': 0}, {'batch_size': 257},
|
||||
{'batch_size': 2.5}, {'normalize': 'true'}, {'query_prefix': 1},
|
||||
{'document_prefix': 'x' * 2001}, {'': 'value'}, {'custom': {}},
|
||||
{'custom': ['value']}, {'x': 1, 'X': 2}, {'custom': float('nan')},
|
||||
{'a' * 65: 'value'}, {str(i): 'value' for i in range(51)}):
|
||||
with self.subTest(params=params), self.assertRaises(ValueError):
|
||||
validate_params(params)
|
||||
|
||||
def test_custom_names_are_saved_but_unsupported_options_do_not_reach_inference(self):
|
||||
params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'}
|
||||
self.assertEqual(validate_params(params), params)
|
||||
active = knowledge_params(SimpleNamespace(meta={'embedding_params': params}))
|
||||
self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:'})
|
||||
base = RecordingModel()
|
||||
model = ConfiguredEmbedding(base, active)
|
||||
model.embed_query('问题')
|
||||
self.assertEqual(base.calls, [('query', '检索:问题')])
|
||||
|
||||
def test_legacy_and_empty_params_keep_original_behavior(self):
|
||||
self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {})
|
||||
self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {})
|
||||
base = RecordingModel()
|
||||
self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [3., 4.])
|
||||
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), [])
|
||||
|
||||
def test_zero_vector_is_safe(self):
|
||||
base = RecordingModel()
|
||||
base.embed_query = lambda _: [0., 0.]
|
||||
self.assertEqual(ConfiguredEmbedding(base, {'normalize': True}).embed_query('x'), [0., 0.])
|
||||
|
|
@ -0,0 +1,288 @@
|
|||
"""Real isolated persistence and document parsing regression checks."""
|
||||
import io
|
||||
import importlib
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch, Mock
|
||||
from django.test import TestCase
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from common.exception.app_exception import AppApiException
|
||||
from common.utils.split_model import smart_split_paragraph, SplitModel
|
||||
from common.utils.table_split import split_table_rows
|
||||
from knowledge.models import KnowledgeFolder, Knowledge
|
||||
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
|
||||
|
||||
|
||||
class MetadataTests(TestCase):
|
||||
def setUp(self):
|
||||
KnowledgeFolder.objects.create(id='default', name='root')
|
||||
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
|
||||
meta={'internal': {'preserve': True}})
|
||||
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
|
||||
|
||||
def test_metadata_persists_and_preserves_other_settings(self):
|
||||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||||
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||||
result = Metadata.one('default', str(self.k.id))
|
||||
self.assertEqual(len(result['fields']), 1)
|
||||
self.assertEqual(result['updated_by'], '测试员')
|
||||
self.assertTrue(result['updated_at'])
|
||||
|
||||
def test_partial_metadata_update_does_not_clear_other_fields(self):
|
||||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||||
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||||
|
||||
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
|
||||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
|
||||
old = Metadata.one('default', str(self.k.id))
|
||||
self.assertEqual(old['fields'][0]['name'], '业务领域')
|
||||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
|
||||
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
|
||||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
|
||||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
|
||||
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
|
||||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||||
|
||||
def test_custom_field_validation_does_not_write(self):
|
||||
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
|
||||
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
|
||||
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
|
||||
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
|
||||
[{'name': str(i), 'value': ''} for i in range(51)]]
|
||||
for fields in cases:
|
||||
with self.subTest(fields=fields), self.assertRaises(AppApiException):
|
||||
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
|
||||
with self.assertRaises(AppApiException):
|
||||
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
|
||||
self.k.refresh_from_db()
|
||||
self.assertNotIn('business_metadata', self.k.meta)
|
||||
|
||||
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
|
||||
from users.models import User
|
||||
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
|
||||
original_time = self.k.create_time
|
||||
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.user_id, creator.id)
|
||||
self.assertEqual(self.k.create_time, original_time)
|
||||
self.assertEqual(result['creator_name'], '新创建者')
|
||||
self.assertEqual(result['user_id'], str(creator.id))
|
||||
self.assertTrue(result['update_time'])
|
||||
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
|
||||
|
||||
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
|
||||
from users.models import User, UserActivationApplication
|
||||
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
|
||||
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
|
||||
for status in ('PENDING', 'REJECTED'):
|
||||
application.status = status; application.save()
|
||||
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||||
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
|
||||
application.status = 'APPROVED'; application.save()
|
||||
creator.is_active = False; creator.save()
|
||||
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
|
||||
self.k.refresh_from_db()
|
||||
self.assertIsNone(self.k.user_id)
|
||||
self.assertNotIn('business_metadata', self.k.meta)
|
||||
|
||||
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
|
||||
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||||
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
|
||||
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||||
operation._validated_data = operation.initial_data
|
||||
operation._errors = {}
|
||||
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
|
||||
with patch.object(operation, 'is_valid'):
|
||||
operation.edit({'embedding_params': params}, select_one=False)
|
||||
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
|
||||
self.k.refresh_from_db(); other.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['embedding_params'], params)
|
||||
self.assertNotIn('embedding_params', other.meta)
|
||||
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
|
||||
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['embedding_params'], params)
|
||||
|
||||
def test_general_settings_cannot_restore_stale_metadata(self):
|
||||
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
|
||||
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
|
||||
self.k.refresh_from_db()
|
||||
protected = self.k.meta['business_metadata']
|
||||
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||||
operation._validated_data = operation.initial_data
|
||||
operation._errors = {}
|
||||
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
|
||||
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
|
||||
self.k.refresh_from_db()
|
||||
self.assertEqual(self.k.meta['business_metadata'], protected)
|
||||
self.assertEqual(self.k.meta['selector'], 'body')
|
||||
self.assertNotIn('parser_id', self.k.meta)
|
||||
|
||||
def test_invalid_types_and_unknown_fields_do_not_write(self):
|
||||
for data in ([1], {'business_domain': 123}, {'business_domain': False},
|
||||
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
|
||||
with self.subTest(data=data), self.assertRaises(AppApiException):
|
||||
Metadata.update('default', str(self.k.id), self.user, data)
|
||||
self.k.refresh_from_db()
|
||||
self.assertNotIn('business_metadata', self.k.meta)
|
||||
|
||||
def test_workspace_boundary(self):
|
||||
for fn in (Metadata.one, Rules.one):
|
||||
with self.assertRaises(AppApiException):
|
||||
fn('other', str(self.k.id))
|
||||
|
||||
def test_segmentation_and_metadata_updates_preserve_each_other(self):
|
||||
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
|
||||
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||||
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
|
||||
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
|
||||
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
|
||||
|
||||
def test_segmentation_rejects_invalid_and_fractional_values(self):
|
||||
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
|
||||
(100,1.2), (True,0), (100,False), ('100.5',0)):
|
||||
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
|
||||
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
|
||||
|
||||
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
|
||||
from knowledge.serializers.document import DocumentSerializers
|
||||
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
|
||||
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
|
||||
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
|
||||
f=SimpleUploadedFile('test.txt',b'test')
|
||||
serializer.parse({'file':[f]})
|
||||
self.assertEqual(split.call_args.args[-2:], (100,20))
|
||||
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
|
||||
self.assertEqual(split.call_args.args[-2:], (200,0))
|
||||
with self.assertRaises(AppApiException):
|
||||
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
|
||||
|
||||
|
||||
class ParserTests(unittest.TestCase):
|
||||
text=''.join(chr(0x4e00+i) for i in range(500))
|
||||
|
||||
def check_overlap(self, parts):
|
||||
self.assertGreater(len(parts),1)
|
||||
self.assertTrue(all(len(p)<=100 for p in parts))
|
||||
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
|
||||
|
||||
def test_plain_text_overlap(self):
|
||||
self.check_overlap(smart_split_paragraph(self.text,100,20))
|
||||
|
||||
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
|
||||
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
|
||||
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
|
||||
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
|
||||
result=PdfSplitHandle.handle_toc(doc,100,20)
|
||||
self.check_overlap([p['content'] for p in result])
|
||||
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
|
||||
import ast,inspect,textwrap
|
||||
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
|
||||
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
|
||||
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
|
||||
self.assertTrue(calls)
|
||||
self.assertTrue(all(len(n.args)==3 for n in calls))
|
||||
|
||||
def test_table_overlap_preserves_rows_headers_and_termination(self):
|
||||
header='| id | value |\n| --- | --- |\n'
|
||||
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
|
||||
chunks=split_table_rows(header,iter(rows),120,20)
|
||||
self.assertGreater(len(chunks),1)
|
||||
self.assertLess(len(chunks),len(rows))
|
||||
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
|
||||
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
|
||||
for a,b in zip(chunks,chunks[1:]):
|
||||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
|
||||
|
||||
def test_oversized_table_row_is_not_truncated(self):
|
||||
row='| '+'x'*500+' |\n'
|
||||
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
|
||||
self.assertTrue(any(row in p['content'] for p in result))
|
||||
self.assertIn('last',result[-1]['content'])
|
||||
|
||||
def test_csv_and_xlsx_use_row_overlap(self):
|
||||
import openpyxl
|
||||
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
|
||||
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
|
||||
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
|
||||
book=openpyxl.Workbook();sheet=book.active
|
||||
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
|
||||
out=io.BytesIO();book.save(out)
|
||||
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
|
||||
f=SimpleUploadedFile(name,data)
|
||||
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
|
||||
chunks=result[0]['content'] if isinstance(result,list) else result['content']
|
||||
self.assertGreater(len(chunks),1)
|
||||
for a,b in zip(chunks,chunks[1:]):
|
||||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||
|
||||
def test_xls_uses_row_overlap(self):
|
||||
from common.handle.impl.text.xls_split_handle import handle_sheet
|
||||
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
|
||||
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
|
||||
chunks=handle_sheet('test',sheet,120,20)['content']
|
||||
for a,b in zip(chunks,chunks[1:]):
|
||||
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||
|
||||
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
|
||||
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
|
||||
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
|
||||
manager=Mock();manager.using.return_value.filter.return_value=[model]
|
||||
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
|
||||
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
|
||||
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
|
||||
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
|
||||
model.save.assert_called_once()
|
||||
|
||||
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
|
||||
import json
|
||||
import httpx
|
||||
from openai import OpenAI
|
||||
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
|
||||
from langchain_core.messages import HumanMessage
|
||||
requests=[]
|
||||
def transport(request):
|
||||
body=json.loads(request.content);requests.append(body)
|
||||
if body.get('stream'):
|
||||
events=[]
|
||||
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
|
||||
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
|
||||
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
|
||||
events.append('data: '+json.dumps(item)+'\n\n')
|
||||
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
|
||||
text=''.join(events)+'data: [DONE]\n\n')
|
||||
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
|
||||
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
|
||||
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
|
||||
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
|
||||
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
|
||||
model=BaiLianChatModel.new_instance('LLM','test',{
|
||||
'api_base':'http://example.invalid/v1','api_key':'test-only'},
|
||||
temperature=0.23,top_p=0.42,max_tokens=10)
|
||||
model.client=client.chat.completions
|
||||
model.root_client=client
|
||||
model.streaming=False
|
||||
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
|
||||
model.streaming=True
|
||||
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
|
||||
self.assertEqual(len(requests),2)
|
||||
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
|
||||
for req in requests:
|
||||
self.assertEqual(req['temperature'],0.23)
|
||||
self.assertEqual(req['top_p'],0.42)
|
||||
|
||||
def test_new_knowledge_tasks_use_isolated_queue(self):
|
||||
from ops import celery_app
|
||||
import knowledge.tasks
|
||||
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
|
||||
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
|
||||
self.assertIn(name,celery_app.tasks)
|
||||
route=celery_app.amqp.router.route({},name)
|
||||
self.assertEqual(route['queue'].name,'uniskb_knowledge')
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls."""
|
||||
import unittest
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch, Mock
|
||||
from contextlib import ExitStack
|
||||
from django.db import transaction, connection
|
||||
from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State
|
||||
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
|
||||
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS
|
||||
from common.exception.app_exception import AppApiException
|
||||
|
||||
|
||||
class RebuildTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute('DELETE FROM knowledge_rebuild_task')
|
||||
self.kid = str(uuid.uuid4())
|
||||
self.docid = str(uuid.uuid4())
|
||||
self.modelid = str(uuid.uuid4())
|
||||
self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test')
|
||||
self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2')
|
||||
self.stack = ExitStack()
|
||||
self.addCleanup(self.stack.close)
|
||||
def mock(name):
|
||||
return self.stack.enter_context(patch(name))
|
||||
self.k = mock('knowledge.serializers.rebuild.Knowledge.objects')
|
||||
self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge
|
||||
self.k.filter.return_value.first.return_value = self.knowledge
|
||||
self.docs = mock('knowledge.serializers.rebuild.Document.objects')
|
||||
self.docs.filter.return_value.values_list.return_value = [self.docid]
|
||||
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document])
|
||||
self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects')
|
||||
self.models = mock('knowledge.serializers.rebuild.Model.objects')
|
||||
self.models.filter.return_value.first.return_value = SimpleNamespace(
|
||||
id=self.modelid, name='embedding', model_name='test-model')
|
||||
self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay')
|
||||
self.delay.return_value.id = 'queued-test'
|
||||
self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async')
|
||||
self.listener = mock('knowledge.task.rebuild.ListenerManagement')
|
||||
self.drop = mock('knowledge.task.rebuild.drop_knowledge_index')
|
||||
self.create_index = mock('knowledge.task.rebuild.create_knowledge_index')
|
||||
self.index_exists = mock('knowledge.task.rebuild._index_exists')
|
||||
self.index_exists.return_value = True
|
||||
self.child = mock('knowledge.task.embedding.embedding_by_document.delay')
|
||||
|
||||
def create(self):
|
||||
return KnowledgeRebuildSerializer.create('default', self.kid, None)
|
||||
|
||||
def task(self, status=RebuildStatus.RUNNING):
|
||||
return KnowledgeRebuildTask.objects.create(
|
||||
knowledge_id=self.kid, workspace_id='default', status=status, document_total=1,
|
||||
config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]})
|
||||
|
||||
def test_dispatch_only_after_commit(self):
|
||||
with transaction.atomic():
|
||||
result = self.create()
|
||||
self.delay.assert_not_called()
|
||||
self.assertEqual(result['status'], RebuildStatus.PENDING)
|
||||
self.delay.assert_called_once()
|
||||
self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test')
|
||||
|
||||
def test_rollback_does_not_dispatch(self):
|
||||
with self.assertRaises(ValueError):
|
||||
with transaction.atomic():
|
||||
self.create()
|
||||
raise ValueError('rollback')
|
||||
self.delay.assert_not_called()
|
||||
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
||||
|
||||
def test_broker_failure_persists_failed_record(self):
|
||||
self.delay.side_effect = RuntimeError('broker offline')
|
||||
result = self.create()
|
||||
self.assertEqual(result['status'], RebuildStatus.FAILED)
|
||||
self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message)
|
||||
|
||||
def test_active_task_rejected_under_knowledge_lock(self):
|
||||
self.task()
|
||||
with self.assertRaises(AppApiException):
|
||||
self.create()
|
||||
self.k.select_for_update.assert_called_once()
|
||||
self.assertEqual(KnowledgeRebuildTask.objects.count(), 1)
|
||||
|
||||
def test_empty_knowledge_rejected(self):
|
||||
self.docs.filter.return_value.values_list.return_value = []
|
||||
with self.assertRaises(AppApiException):
|
||||
self.create()
|
||||
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
||||
|
||||
def test_success_cannot_be_retried(self):
|
||||
task = self.task(RebuildStatus.SUCCESS)
|
||||
with self.assertRaises(AppApiException):
|
||||
KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None)
|
||||
|
||||
def test_failed_retry_creates_new_record(self):
|
||||
old = self.task(RebuildStatus.FAILED)
|
||||
result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None)
|
||||
self.assertNotEqual(result['id'], str(old.id))
|
||||
self.assertEqual(KnowledgeRebuildTask.objects.count(), 2)
|
||||
|
||||
def test_duplicate_worker_delivery_does_not_delete_vectors(self):
|
||||
task = self.task()
|
||||
rebuild_knowledge_index.run(str(task.id))
|
||||
self.listener.delete_embedding_by_knowledge.assert_not_called()
|
||||
|
||||
def test_worker_dispatches_independent_monitor(self):
|
||||
task = self.task(RebuildStatus.PENDING)
|
||||
rebuild_knowledge_index.run(str(task.id))
|
||||
self.child.assert_called_once_with(self.docid, self.modelid)
|
||||
self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid])
|
||||
self.poll.assert_called_once()
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
||||
|
||||
def test_document_dispatch_failure_is_counted(self):
|
||||
task = self.task(RebuildStatus.PENDING)
|
||||
self.child.side_effect = RuntimeError('cannot queue')
|
||||
rebuild_knowledge_index.run(str(task.id))
|
||||
self.listener.update_status.assert_any_call(
|
||||
self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE)
|
||||
self.document.status = 'nn3'
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
self.assertEqual(task.document_failed, 1)
|
||||
self.assertIn('cannot queue', task.error_message)
|
||||
|
||||
def test_success_requires_index(self):
|
||||
task = self.task()
|
||||
self.index_exists.return_value = False
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
|
||||
def test_successful_finalization(self):
|
||||
task = self.task()
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.SUCCESS)
|
||||
self.assertEqual(task.document_success, 1)
|
||||
self.assertIsNotNone(task.finished_at)
|
||||
|
||||
def test_pending_persists_progress_and_schedules_next_poll(self):
|
||||
task = self.task()
|
||||
other = SimpleNamespace(id=uuid.uuid4(), status='nn0')
|
||||
task.config_snapshot['document_ids'].append(str(other.id))
|
||||
task.save()
|
||||
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other])
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
||||
self.assertEqual(task.document_success, 1)
|
||||
self.poll.assert_called_once()
|
||||
|
||||
def test_timeout_records_failure(self):
|
||||
task = self.task()
|
||||
task.attempt = MAX_POLLS - 1
|
||||
task.save()
|
||||
self.document.status = 'nn0'
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
self.assertIn('超时', task.error_message)
|
||||
|
||||
def test_ignored_document_is_failure(self):
|
||||
task = self.task()
|
||||
self.document.status = 'nnn'
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.document_failed, 1)
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
|
||||
def test_deleted_document_is_not_silent_success(self):
|
||||
task = self.task()
|
||||
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([])
|
||||
finalize_rebuild_task.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.document_failed, 1)
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
|
||||
def test_model_change_fails_before_destructive_work(self):
|
||||
task = self.task(RebuildStatus.PENDING)
|
||||
self.knowledge.embedding_model_id = str(uuid.uuid4())
|
||||
rebuild_knowledge_index.run(str(task.id))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||
self.listener.delete_embedding_by_knowledge.assert_not_called()
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
"""Weight owner for local resources; shared by both assistants through model IDs."""
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import requests
|
||||
from django.db import close_old_connections
|
||||
|
||||
from common.model_runtime import ModelRuntime, loading_policy
|
||||
|
||||
logger = logging.getLogger('maxkb')
|
||||
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
|
||||
|
||||
|
||||
def driver_for(provider, model_type):
|
||||
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
|
||||
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
|
||||
|
||||
|
||||
def model_spec(model_id):
|
||||
from local_model.models import Model
|
||||
from local_model.serializers.rsa_util import rsa_long_decrypt
|
||||
try:
|
||||
model = Model.objects.get(id=model_id, status='SUCCESS')
|
||||
except Model.DoesNotExist:
|
||||
raise ValueError('模型不存在或尚不可用')
|
||||
driver = driver_for(model.provider, model.model_type)
|
||||
if not driver:
|
||||
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
|
||||
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
|
||||
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
|
||||
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
|
||||
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
|
||||
|
||||
|
||||
class WeightAdapter:
|
||||
@staticmethod
|
||||
def _request(spec, route, data=None):
|
||||
url = urlsplit(spec['credential'].get('api_base', ''))
|
||||
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
|
||||
raise ValueError('模型服务地址无效')
|
||||
path = url.path.rstrip('/')
|
||||
if path.endswith('/v1'):
|
||||
path = path[:-3]
|
||||
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
|
||||
key = spec['credential'].get('api_key')
|
||||
headers = {'Authorization': 'Bearer ' + key} if key else {}
|
||||
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
|
||||
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def probe(self, spec):
|
||||
try:
|
||||
tags = self._request(spec, '/api/ps').get('models', [])
|
||||
except requests.RequestException as exc:
|
||||
raise ValueError('模型推理服务不可用') from exc
|
||||
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
|
||||
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
|
||||
if item:
|
||||
vram = item.get('size_vram', 0)
|
||||
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
|
||||
|
||||
def load(self, spec):
|
||||
close_old_connections()
|
||||
try:
|
||||
if spec['driver'] == 'ollama':
|
||||
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
|
||||
if spec['type'] == 'EMBEDDING':
|
||||
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
|
||||
else:
|
||||
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
|
||||
'keep_alive': keep_alive})
|
||||
info = self.probe(spec)
|
||||
if not info:
|
||||
raise ValueError('模型未驻留')
|
||||
logger.info('Model weights ready: %s', spec['id'])
|
||||
return True, info
|
||||
from local_model.serializers.model_apply_serializers import get_local_model
|
||||
instance = get_local_model(spec['model'], use_local=True)
|
||||
if spec['type'] == 'EMBEDDING':
|
||||
instance.embed_query('模型加载检查')
|
||||
else:
|
||||
from langchain_core.documents import Document
|
||||
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
|
||||
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
|
||||
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
|
||||
info = {'device': device}
|
||||
try:
|
||||
import torch
|
||||
if device.startswith('cuda'):
|
||||
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
|
||||
except ImportError:
|
||||
pass
|
||||
logger.info('Model weights ready: %s', spec['id'])
|
||||
return instance, info
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
||||
def unload(self, spec, value):
|
||||
if spec['driver'] == 'ollama':
|
||||
# Refuse to unload a native model shared by another configured resource.
|
||||
from local_model.models import Model
|
||||
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
|
||||
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
|
||||
if spec['type'] == 'EMBEDDING':
|
||||
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
|
||||
else:
|
||||
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
|
||||
for _ in range(50):
|
||||
if self.probe(spec) is None:
|
||||
return
|
||||
time.sleep(.2)
|
||||
raise ValueError('模型卸载未完成,请稍后重试')
|
||||
|
||||
@staticmethod
|
||||
def collect():
|
||||
gc.collect()
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_initialized():
|
||||
torch.cuda.empty_cache()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def error(exc):
|
||||
name = type(exc).__name__
|
||||
logger.warning('Model weight load failed: %s', name)
|
||||
if 'OutOfMemory' in name:
|
||||
return '模型加载失败:显存或内存不足'
|
||||
return '模型加载失败,请检查模型文件、运行设备及服务状态'
|
||||
|
||||
|
||||
manager = ModelRuntime(model_spec, WeightAdapter())
|
||||
_startup_lock = threading.Lock()
|
||||
_started = False
|
||||
|
||||
|
||||
def start_runtime():
|
||||
global _started
|
||||
with _startup_lock:
|
||||
if _started:
|
||||
return
|
||||
_started = True
|
||||
def run():
|
||||
from django.apps import apps
|
||||
while not apps.ready:
|
||||
time.sleep(.05)
|
||||
from local_model.models import Model
|
||||
try:
|
||||
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
|
||||
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
|
||||
manager.preload(ids)
|
||||
logger.info('Startup model preloads queued: %s', len(ids))
|
||||
except Exception as exc:
|
||||
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
|
||||
finally:
|
||||
close_old_connections()
|
||||
while True:
|
||||
time.sleep(10)
|
||||
try:
|
||||
manager.sweep()
|
||||
except Exception as exc:
|
||||
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
|
||||
finally:
|
||||
close_old_connections()
|
||||
threading.Thread(target=run, name='model-startup', daemon=True).start()
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
import re
|
||||
|
||||
from rest_framework.exceptions import PermissionDenied, ValidationError
|
||||
from rest_framework.serializers import UUIDField
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from common.model_runtime_client import verify_request
|
||||
from common.auth.authenticate import AnonymousAuthentication
|
||||
from common.result import result
|
||||
from local_model.runtime import manager
|
||||
|
||||
|
||||
class LocalModelRuntime(APIView):
|
||||
authentication_classes = [AnonymousAuthentication]
|
||||
permission_classes = []
|
||||
|
||||
def post(self, request):
|
||||
try:
|
||||
data = verify_request(request.body, request.headers.get('X-Model-Runtime-Signature'))
|
||||
except (ValueError, TypeError):
|
||||
raise PermissionDenied('模型服务认证失败')
|
||||
model_id = str(UUIDField().run_validation(data.get('model_id')))
|
||||
operation = data.get('operation')
|
||||
try:
|
||||
if operation == 'release':
|
||||
lease = data.get('lease', '')
|
||||
if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{32}', lease):
|
||||
raise ValueError('请求标识无效')
|
||||
manager.release(model_id, lease)
|
||||
return result.success({'released': True})
|
||||
if operation == 'acquire':
|
||||
lease = manager.acquire(model_id, remote=True)
|
||||
try:
|
||||
return result.success({**manager.status(model_id), 'lease': lease})
|
||||
except Exception:
|
||||
manager.release(model_id, lease)
|
||||
raise
|
||||
if operation == 'load':
|
||||
manager.submit(model_id)
|
||||
elif operation == 'unload':
|
||||
manager.unload(model_id)
|
||||
elif operation == 'configure':
|
||||
manager.configure(model_id)
|
||||
elif operation != 'status':
|
||||
raise ValueError('模型服务操作无效')
|
||||
return result.success(manager.status(model_id))
|
||||
except (ValueError, TimeoutError) as exc:
|
||||
raise ValidationError(str(exc))
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
"""Model file assets, independent of runtime/model connection records. Never executes model code."""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
MAX_BYTES = 2 * 1024 ** 3
|
||||
MAX_FILES = 10000
|
||||
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
|
||||
|
||||
|
||||
class ImportError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def inspect_model(directory):
|
||||
root = Path(directory)
|
||||
if not root.is_dir() or root.is_symlink():
|
||||
raise ImportError('请选择有效模型目录')
|
||||
files = []
|
||||
for current, dirs, names in os.walk(root, followlinks=False):
|
||||
for name in dirs + names:
|
||||
if (Path(current) / name).is_symlink():
|
||||
raise ImportError('模型目录不能包含符号链接')
|
||||
for name in names:
|
||||
file = Path(current) / name
|
||||
if not file.is_file():
|
||||
raise ImportError('模型目录包含非普通文件')
|
||||
files.append(file)
|
||||
if len(files) > MAX_FILES:
|
||||
raise ImportError('模型文件数量超过限制')
|
||||
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
|
||||
if not weights:
|
||||
raise ImportError('未找到有效模型权重文件')
|
||||
|
||||
def read_json(name):
|
||||
file = root / name
|
||||
if not file.exists():
|
||||
return {}
|
||||
if file.stat().st_size > 2 * 1024 ** 2:
|
||||
raise ImportError(name + '超过大小限制')
|
||||
try:
|
||||
data = json.loads(file.read_text(encoding='utf-8'))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError()
|
||||
return data
|
||||
except (ValueError, UnicodeError):
|
||||
raise ImportError(name + '格式错误')
|
||||
|
||||
config = read_json('config.json')
|
||||
manifest = read_json('model-manifest.json')
|
||||
quant = config.get('quantization_config') or {}
|
||||
if not isinstance(quant, dict):
|
||||
raise ImportError('量化配置格式错误')
|
||||
quantization = str(quant.get('quant_method', '')).lower()
|
||||
suffixes = {f.suffix.lower() for f in weights}
|
||||
if config and suffixes & {'.safetensors', '.bin'}:
|
||||
if not config.get('model_type') and not config.get('architectures'):
|
||||
raise ImportError('config.json缺少模型架构信息')
|
||||
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
|
||||
elif '.onnx' in suffixes:
|
||||
format_name = 'ONNX'
|
||||
elif '.gguf' in suffixes:
|
||||
format_name = 'GGUF'
|
||||
elif manifest:
|
||||
format_name = 'CUSTOM'
|
||||
else:
|
||||
raise ImportError('模型目录缺少config.json或model-manifest.json')
|
||||
# Validate referenced shards without trusting paths supplied by model files.
|
||||
for index in root.glob('*.index.json'):
|
||||
mapping = read_json(index.name).get('weight_map')
|
||||
if not isinstance(mapping, dict) or not mapping:
|
||||
raise ImportError('权重索引格式错误')
|
||||
for name in mapping.values():
|
||||
if not isinstance(name, str) or '\\' in name:
|
||||
raise ImportError('权重分片路径无效')
|
||||
relative = PurePosixPath(name)
|
||||
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
|
||||
raise ImportError('权重分片缺失或路径无效')
|
||||
return {
|
||||
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
|
||||
'file_count': len(files), 'weight_count': len(weights),
|
||||
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
|
||||
'quantization': quantization,
|
||||
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
|
||||
'runtime_status': 'NOT_CHECKED',
|
||||
'runtime_note': '文件已导入,运行能力需接入验证',
|
||||
}
|
||||
|
||||
|
||||
class ModelImportStore:
|
||||
def __init__(self, root):
|
||||
self.root = Path(root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def list(self):
|
||||
records = []
|
||||
for path in self.root.glob('*/import.json'):
|
||||
try:
|
||||
records.append(json.loads(path.read_text(encoding='utf-8')))
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
return sorted(records, key=lambda r: r['created_at'], reverse=True)[:200]
|
||||
|
||||
def create(self, owner_id, path=None, upload=None):
|
||||
if (path is None) == (upload is None):
|
||||
raise ImportError('请选择一种导入方式')
|
||||
identity = str(uuid.uuid4())
|
||||
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
|
||||
try:
|
||||
if upload is not None:
|
||||
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
|
||||
raise ImportError('请上传2GB以内的ZIP模型包')
|
||||
archive = staging / 'upload.zip'
|
||||
with archive.open('wb') as output:
|
||||
size = 0
|
||||
for chunk in upload.chunks():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise ImportError('模型包超过2GB')
|
||||
output.write(chunk)
|
||||
source = staging / 'files'
|
||||
source.mkdir()
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as zipped:
|
||||
members = zipped.infolist()
|
||||
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
|
||||
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
|
||||
seen = set()
|
||||
for member in members:
|
||||
name = PurePosixPath(member.filename)
|
||||
mode = member.external_attr >> 16
|
||||
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
|
||||
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
|
||||
or member.filename in seen):
|
||||
raise ImportError('模型包包含不安全的路径或文件')
|
||||
seen.add(member.filename)
|
||||
target = source.joinpath(*name.parts)
|
||||
if member.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipped.open(member) as incoming, target.open('xb') as output:
|
||||
shutil.copyfileobj(incoming, output, 1024 * 1024)
|
||||
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
|
||||
raise ImportError('模型包损坏或无法解压') from exc
|
||||
archive.unlink()
|
||||
# Most model archives contain one enclosing directory.
|
||||
children = list(source.iterdir())
|
||||
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
|
||||
info = inspect_model(model_dir)
|
||||
final_path = self.root / identity / model_dir.relative_to(staging)
|
||||
source_type = 'UPLOAD'
|
||||
display_name = Path(upload.name).stem
|
||||
else:
|
||||
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
|
||||
raise ImportError('请输入服务器绝对目录')
|
||||
model_dir = Path(path).resolve(strict=True)
|
||||
info = inspect_model(model_dir)
|
||||
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
|
||||
record = {
|
||||
'id': identity, 'name': display_name, 'path': str(final_path),
|
||||
'source': source_type, 'owner_id': str(owner_id),
|
||||
'created_at': datetime.now(timezone.utc).isoformat(),
|
||||
'status': 'IMPORTED', **info,
|
||||
}
|
||||
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
|
||||
staging.rename(self.root / identity)
|
||||
return record
|
||||
except (FileNotFoundError, PermissionError) as exc:
|
||||
raise ImportError('目录不存在或无访问权限') from exc
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""补齐已有百炼模型表单中的 TopP;保留全部自定义参数项。"""
|
||||
from django.db import migrations
|
||||
|
||||
TOP_P_FIELD = {
|
||||
"input_type": "Slider", "label": "TopP", "required": True,
|
||||
"default_value": 0.8, "relation_show_field_dict": {},
|
||||
"relation_trigger_field_dict": {}, "trigger_type": "OPTION_LIST",
|
||||
"attrs": {"min": 0, "max": 1, "step": 0.01, "precision": 2,
|
||||
"show-input-controls": False, "show-input": True},
|
||||
"props_info": {}, "field": "top_p",
|
||||
}
|
||||
|
||||
|
||||
def add_top_p_params_form(apps, schema_editor):
|
||||
Model = apps.get_model('models_provider', 'Model')
|
||||
for model in Model.objects.using(schema_editor.connection.alias).filter(
|
||||
provider='aliyun_bai_lian_model_provider', model_type='LLM'):
|
||||
form = model.model_params_form or []
|
||||
if any(item.get('field') == 'top_p' for item in form):
|
||||
continue
|
||||
model.model_params_form = [*form, TOP_P_FIELD]
|
||||
model.save(update_fields=['model_params_form'], using=schema_editor.connection.alias)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [('models_provider', '0001_initial')]
|
||||
# 回退代码时保留参数项,不删除可能由管理员维护的数据。
|
||||
operations = [migrations.RunPython(add_top_p_params_form, migrations.RunPython.noop)]
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
"""File import regression tests, temporary files only; no database or model runtimes."""
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
|
||||
|
||||
|
||||
class Upload:
|
||||
def __init__(self, entries):
|
||||
data = io.BytesIO()
|
||||
with zipfile.ZipFile(data, 'w') as archive:
|
||||
for name, content in entries:
|
||||
archive.writestr(name, content)
|
||||
self.body = data.getvalue()
|
||||
self.size, self.name = len(self.body), 'model.zip'
|
||||
|
||||
def chunks(self):
|
||||
yield self.body
|
||||
|
||||
|
||||
class ImportTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.model = self.root / 'model'
|
||||
self.model.mkdir()
|
||||
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
|
||||
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
|
||||
self.store = ModelImportStore(self.root / 'imports')
|
||||
|
||||
def test_local_path_record_persists_without_copying_or_running(self):
|
||||
record = self.store.create('owner', path=str(self.model))
|
||||
self.assertEqual(record['format'], 'HF')
|
||||
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
|
||||
self.assertEqual(record['path'], str(self.model.resolve()))
|
||||
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
|
||||
|
||||
def test_quantized_formats(self):
|
||||
for method in ('gptq', 'awq'):
|
||||
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
|
||||
self.assertEqual(inspect_model(self.model)['format'], method.upper())
|
||||
|
||||
def test_onnx_gguf_custom(self):
|
||||
(self.model / 'config.json').unlink()
|
||||
(self.model / 'model.safetensors').unlink()
|
||||
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
|
||||
file = self.model / ('model' + suffix)
|
||||
file.write_bytes(b'fixture')
|
||||
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
|
||||
self.assertEqual(inspect_model(self.model)['format'], expected)
|
||||
file.unlink()
|
||||
|
||||
def test_zip_nested_model_and_persisted_path(self):
|
||||
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
|
||||
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
|
||||
self.assertEqual(record['source'], 'UPLOAD')
|
||||
|
||||
def test_traversal_windows_path_rejected(self):
|
||||
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
|
||||
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
|
||||
self.assertEqual(self.store.list(), [])
|
||||
|
||||
def test_duplicate_archive_entry_rejected(self):
|
||||
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
|
||||
|
||||
def test_zip_symlink_rejected(self):
|
||||
data=io.BytesIO()
|
||||
with zipfile.ZipFile(data,'w') as archive:
|
||||
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
|
||||
archive.writestr(info, '../outside')
|
||||
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
|
||||
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||
|
||||
def test_size_limit_and_cleanup(self):
|
||||
upload=Upload([('model.onnx', b'x'*100)])
|
||||
with patch('models_provider.local_imports.MAX_BYTES', 20):
|
||||
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||
self.assertEqual(list(self.store.root.iterdir()), [])
|
||||
|
||||
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
|
||||
(self.model / 'config.json').write_text('{broken')
|
||||
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||
(self.model / 'config.json').write_text('{"other":1}')
|
||||
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||
(self.model / 'model.safetensors').unlink()
|
||||
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||
|
||||
def test_missing_or_escaping_shard_rejected(self):
|
||||
for name in ('absent.safetensors','../outside.bin'):
|
||||
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
|
||||
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||
|
||||
def test_bad_zip_and_missing_path_leave_no_records(self):
|
||||
upload=Upload([]);upload.body=b'not zip';upload.size=7
|
||||
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
|
||||
self.assertEqual(self.store.list(), [])
|
||||
|
||||
def test_exclusive_source(self):
|
||||
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
|
||||
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.test import TestCase
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
from common.constants.permission_constants import RoleConstants
|
||||
from common.exception.app_exception import AppUnauthorizedFailed
|
||||
from models_provider.models import Model
|
||||
from models_provider.views.runtime import ModelLoading
|
||||
|
||||
|
||||
class LoadingApiTests(TestCase):
|
||||
def setUp(self):
|
||||
self.model = Model.objects.create(name='loading-test', provider='model_local_provider',
|
||||
model_type='EMBEDDING', model_name='test-only', credential='test-only', meta={'existing': 'keep'})
|
||||
self.admin = SimpleNamespace(auth=SimpleNamespace(role_list=[str(RoleConstants.ADMIN.value)], permission_list=[]), data={})
|
||||
self.view = ModelLoading()
|
||||
|
||||
def args(self):
|
||||
return {'workspace_id': 'default', 'model_id': str(self.model.id)}
|
||||
|
||||
def test_policy_is_persistent_and_preserves_existing_metadata(self):
|
||||
self.admin.data = {'strategy': 'PRELOAD', 'priority': 90}
|
||||
with patch('models_provider.views.runtime.runtime_request', return_value={'status': 'QUEUED'}) as rpc:
|
||||
response = self.view.put(self.admin, **self.args())
|
||||
self.model.refresh_from_db()
|
||||
self.assertEqual(self.model.meta['existing'], 'keep')
|
||||
self.assertEqual(self.model.meta['loading_policy'], {'strategy': 'PRELOAD', 'priority': 90, 'idle_seconds': 300})
|
||||
self.assertEqual(json.loads(response.content)['data']['status'], 'QUEUED')
|
||||
self.assertEqual(rpc.call_args_list[0].args[0], 'configure')
|
||||
|
||||
def test_normal_user_cannot_change_policy_or_load_weights(self):
|
||||
user = SimpleNamespace(auth=SimpleNamespace(role_list=[], permission_list=[]), data={'action': 'load'})
|
||||
for method in (self.view.get, self.view.put, self.view.post):
|
||||
with self.assertRaises(AppUnauthorizedFailed):
|
||||
method(user, **self.args())
|
||||
|
||||
def test_workspace_and_unsupported_provider_are_rejected(self):
|
||||
with self.assertRaises(ValidationError):
|
||||
self.view.get(self.admin, workspace_id='other', model_id=str(self.model.id))
|
||||
self.model.provider = 'aliyun_bai_lian_model_provider'
|
||||
self.model.model_type = 'LLM'; self.model.save()
|
||||
response = self.view.get(self.admin, **self.args())
|
||||
self.assertEqual(json.loads(response.content)['data']['status'], 'EXTERNAL')
|
||||
with self.assertRaises(ValidationError):
|
||||
self.view.put(self.admin, **self.args())
|
||||
|
||||
def test_arbitrary_actions_and_invalid_policies_do_not_reach_runtime(self):
|
||||
with patch('models_provider.views.runtime.runtime_request') as rpc:
|
||||
self.admin.data = {'action': 'shell'}
|
||||
with self.assertRaises(ValidationError): self.view.post(self.admin, **self.args())
|
||||
self.admin.data = {'priority': 500}
|
||||
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
|
||||
rpc.assert_not_called()
|
||||
self.model.refresh_from_db()
|
||||
self.assertNotIn('loading_policy', self.model.meta)
|
||||
|
||||
def test_saved_policy_remains_available_when_runtime_is_temporarily_down(self):
|
||||
self.admin.data = {'strategy': 'PRELOAD', 'priority': 80}
|
||||
with patch('models_provider.views.runtime.runtime_request', side_effect=ValueError('offline')):
|
||||
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
|
||||
response = self.view.get(self.admin, **self.args())
|
||||
self.model.refresh_from_db()
|
||||
self.assertEqual(self.model.meta['loading_policy']['priority'], 80)
|
||||
self.assertEqual(json.loads(response.content)['data']['status'], 'UNAVAILABLE')
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
import contextlib
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from common.model_runtime import ModelRuntime, loading_policy
|
||||
from common.model_runtime_client import signed_request, verify_request
|
||||
|
||||
|
||||
class FakeAdapter:
|
||||
def __init__(self):
|
||||
self.loaded, self.unloaded = [], []
|
||||
self.native_ready = True
|
||||
self.fail = False
|
||||
self.block = None
|
||||
|
||||
def load(self, spec):
|
||||
if self.block:
|
||||
self.block.wait(3)
|
||||
if self.fail:
|
||||
raise ValueError('private error details')
|
||||
self.loaded.append(spec['id'])
|
||||
self.native_ready = True
|
||||
return SimpleNamespace(), {'device': 'cpu'}
|
||||
|
||||
def unload(self, spec, value):
|
||||
self.unloaded.append(spec['id'])
|
||||
self.native_ready = False
|
||||
|
||||
def probe(self, spec):
|
||||
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
|
||||
|
||||
def collect(self):
|
||||
pass
|
||||
|
||||
def error(self, exc):
|
||||
return 'load failed'
|
||||
|
||||
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.now = 0
|
||||
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
|
||||
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
|
||||
for key, priority in [('low', 10), ('high', 90)]}
|
||||
self.adapter = FakeAdapter()
|
||||
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
|
||||
self.adapter, lambda: self.now)
|
||||
|
||||
def test_no_load_until_first_inference_and_then_reuses_weights(self):
|
||||
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
||||
self.assertEqual(self.adapter.loaded, [])
|
||||
with self.manager.inference('low') as first:
|
||||
self.assertIsNotNone(first)
|
||||
with self.manager.inference('low') as second:
|
||||
self.assertIs(first, second)
|
||||
self.assertEqual(self.adapter.loaded, ['low'])
|
||||
|
||||
def test_startup_preloads_highest_priority_first(self):
|
||||
self.specs['low']['policy']['strategy'] = 'PRELOAD'
|
||||
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
||||
self.manager.preload(['low', 'high'])
|
||||
for key in ['low', 'high']:
|
||||
self.manager.entries[key]['future'].result(3)
|
||||
self.assertEqual(self.adapter.loaded, ['high', 'low'])
|
||||
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
|
||||
|
||||
def test_concurrent_first_calls_only_load_once(self):
|
||||
self.adapter.block = threading.Event()
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
|
||||
self.adapter.block.set()
|
||||
tokens = [future.result(3) for future in futures]
|
||||
self.assertEqual(len(set(tokens)), 6)
|
||||
self.assertEqual(self.adapter.loaded, ['low'])
|
||||
self.assertEqual(self.manager.status('low')['active_requests'], 6)
|
||||
for token in tokens:
|
||||
self.manager.release('low', token)
|
||||
|
||||
def test_in_use_models_cannot_be_unloaded(self):
|
||||
token = self.manager.acquire('low')
|
||||
with self.assertRaises(ValueError):
|
||||
self.manager.unload('low')
|
||||
self.manager.release('low', token)
|
||||
self.manager.unload('low')
|
||||
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
||||
|
||||
def test_unload_releases_the_last_loader_reference(self):
|
||||
class Weights:
|
||||
pass
|
||||
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
|
||||
token = self.manager.acquire('low')
|
||||
pointer = weakref.ref(self.manager.entries['low']['value'])
|
||||
self.manager.release('low', token)
|
||||
self.manager.unload('low')
|
||||
self.assertIsNone(pointer())
|
||||
|
||||
def test_idle_cleanup_respects_active_and_preloaded_models(self):
|
||||
active = self.manager.acquire('low')
|
||||
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
||||
self.manager.preload(['high'])
|
||||
self.manager.entries['high']['future'].result(3)
|
||||
self.now = 100
|
||||
self.manager.sweep()
|
||||
self.assertEqual(self.adapter.unloaded, [])
|
||||
self.manager.release('low', active)
|
||||
self.now = 131
|
||||
self.manager.sweep()
|
||||
self.assertEqual(self.adapter.unloaded, ['low'])
|
||||
self.assertEqual(self.manager.status('high')['status'], 'READY')
|
||||
|
||||
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
|
||||
self.adapter.fail = True
|
||||
with self.assertRaisesRegex(ValueError, '^load failed$'):
|
||||
self.manager.acquire('low')
|
||||
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
|
||||
self.adapter.fail = False
|
||||
with self.manager.inference('low'):
|
||||
pass
|
||||
self.assertEqual(self.manager.status('low')['status'], 'READY')
|
||||
|
||||
def test_changed_model_config_never_reuses_old_weights(self):
|
||||
token = self.manager.acquire('low')
|
||||
self.specs['low']['fingerprint'] = 'v2'
|
||||
with self.assertRaises(ValueError):
|
||||
self.manager.acquire('low')
|
||||
self.manager.release('low', token)
|
||||
with self.manager.inference('low'):
|
||||
pass
|
||||
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
||||
self.assertEqual(self.adapter.unloaded, ['low'])
|
||||
|
||||
def test_native_expiration_requires_loading_again(self):
|
||||
self.specs['low']['driver'] = 'ollama'
|
||||
token = self.manager.acquire('low')
|
||||
self.manager.release('low', token)
|
||||
self.adapter.native_ready = False
|
||||
token = self.manager.acquire('low')
|
||||
self.manager.release('low', token)
|
||||
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
||||
|
||||
def test_policy_validation(self):
|
||||
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
|
||||
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
|
||||
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||
loading_policy(value)
|
||||
|
||||
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
|
||||
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
|
||||
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
|
||||
self.assertEqual(verify_request(body, signature)['operation'], 'load')
|
||||
with self.assertRaises(ValueError):
|
||||
verify_request(body.replace(b'load', b'unload'), signature)
|
||||
with self.assertRaises(ValueError):
|
||||
verify_request(body, '')
|
||||
with patch('common.model_runtime_client.time.time', return_value=1):
|
||||
body, signature = signed_request({'operation': 'load'})
|
||||
with self.assertRaises(ValueError):
|
||||
verify_request(body, signature)
|
||||
|
||||
|
||||
class ProviderHookTests(unittest.TestCase):
|
||||
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
|
||||
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
|
||||
events = []
|
||||
@contextlib.contextmanager
|
||||
def lease(model_id):
|
||||
events.append(('acquire', model_id))
|
||||
try:
|
||||
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
||||
finally:
|
||||
events.append(('release', model_id))
|
||||
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
||||
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
|
||||
self.assertEqual(model.embed_query('text'), [1., 2.])
|
||||
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
|
||||
self.assertEqual(model._default_params['keep_alive'], -1)
|
||||
|
||||
def test_llm_stream_releases_lease_when_generator_is_closed(self):
|
||||
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
|
||||
from langchain_ollama import ChatOllama
|
||||
events = []
|
||||
@contextlib.contextmanager
|
||||
def lease(model_id):
|
||||
events.append('acquire')
|
||||
try:
|
||||
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
||||
finally:
|
||||
events.append('release')
|
||||
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
||||
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
|
||||
stream = model._create_chat_stream([])
|
||||
next(stream); stream.close()
|
||||
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
|
||||
self.assertEqual(events, ['acquire', 'release'])
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from rest_framework.views import APIView
|
||||
from common.auth import TokenAuth
|
||||
from common.auth.authentication import has_permissions
|
||||
from common.constants.permission_constants import RoleConstants
|
||||
from common.exception.app_exception import AppApiException
|
||||
from common.result import result
|
||||
from maxkb.conf import PROJECT_DIR
|
||||
from models_provider.local_imports import ModelImportStore, ImportError
|
||||
|
||||
|
||||
class LocalModelImports(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@staticmethod
|
||||
def store():
|
||||
return ModelImportStore(os.environ.get('MAXKB_MODEL_IMPORT_DIR', str(Path(PROJECT_DIR) / 'model' / 'imports')))
|
||||
|
||||
@has_permissions(RoleConstants.ADMIN)
|
||||
def get(self, request):
|
||||
return result.success(self.store().list())
|
||||
|
||||
@has_permissions(RoleConstants.ADMIN)
|
||||
def post(self, request):
|
||||
try:
|
||||
return result.success(self.store().create(request.user.id,
|
||||
path=request.data.get('path'), upload=request.FILES.get('file')))
|
||||
except ImportError as exc:
|
||||
raise AppApiException(500, str(exc))
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
from django.db import transaction
|
||||
from rest_framework.exceptions import ValidationError
|
||||
from rest_framework.serializers import UUIDField
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from common.auth import TokenAuth
|
||||
from common.auth.authentication import has_permissions
|
||||
from common.constants.permission_constants import RoleConstants
|
||||
from common.model_runtime import loading_policy
|
||||
from common.model_runtime_client import runtime_request
|
||||
from common.result import result
|
||||
from models_provider.models import Model
|
||||
|
||||
|
||||
def managed_model(model):
|
||||
return ((model.provider == 'model_local_provider' and model.model_type in ('EMBEDDING', 'RERANKER')) or
|
||||
(model.provider == 'model_ollama_provider' and model.model_type in ('EMBEDDING', 'LLM')))
|
||||
|
||||
|
||||
class ModelLoading(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@staticmethod
|
||||
def model(workspace_id, model_id):
|
||||
model_id = UUIDField().run_validation(model_id)
|
||||
model = Model.objects.filter(id=model_id, workspace_id=workspace_id).first()
|
||||
if model is None:
|
||||
raise ValidationError('模型不存在')
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def state(model):
|
||||
policy = loading_policy((model.meta or {}).get('loading_policy'))
|
||||
if not managed_model(model):
|
||||
return {**policy, 'supported': False, 'status': 'EXTERNAL'}
|
||||
try:
|
||||
return {**runtime_request('status', model.id), 'supported': True}
|
||||
except ValueError:
|
||||
return {**policy, 'supported': True, 'status': 'UNAVAILABLE', 'error': '本地模型服务不可用'}
|
||||
|
||||
@has_permissions(RoleConstants.ADMIN)
|
||||
def get(self, request, workspace_id, model_id):
|
||||
return result.success(self.state(self.model(workspace_id, model_id)))
|
||||
|
||||
@has_permissions(RoleConstants.ADMIN)
|
||||
def put(self, request, workspace_id, model_id):
|
||||
model = self.model(workspace_id, model_id)
|
||||
if not managed_model(model):
|
||||
raise ValidationError('该模型由外部服务运行')
|
||||
try:
|
||||
policy = loading_policy(request.data)
|
||||
except ValueError as exc:
|
||||
raise ValidationError(str(exc))
|
||||
with transaction.atomic():
|
||||
model = Model.objects.select_for_update().get(id=model.id)
|
||||
model.meta = {**(model.meta or {}), 'loading_policy': policy}
|
||||
model.save(update_fields=['meta', 'update_time'])
|
||||
try:
|
||||
runtime_request('configure', model.id)
|
||||
except ValueError:
|
||||
raise ValidationError('策略已保存,本地模型服务暂不可用,请恢复服务后重试')
|
||||
return result.success(self.state(model))
|
||||
|
||||
@has_permissions(RoleConstants.ADMIN)
|
||||
def post(self, request, workspace_id, model_id):
|
||||
model = self.model(workspace_id, model_id)
|
||||
if not managed_model(model):
|
||||
raise ValidationError('该模型由外部服务运行')
|
||||
operation = request.data.get('action')
|
||||
if operation not in ('load', 'unload'):
|
||||
raise ValidationError('无效操作')
|
||||
try:
|
||||
return result.success({**runtime_request(operation, model.id), 'supported': True})
|
||||
except ValueError as exc:
|
||||
raise ValidationError(str(exc))
|
||||
|
|
@ -0,0 +1,453 @@
|
|||
"""AI hub business operations. Secrets never cross the browser boundary."""
|
||||
from urllib.parse import urlsplit
|
||||
from django.db import transaction
|
||||
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
|
||||
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
|
||||
|
||||
MAX_TEXT = 2_000_000
|
||||
MAX_INLINE_TEXT = MAX_TEXT + 140_000_000
|
||||
MAX_FILE = 20 * 1024 * 1024
|
||||
KINDS = {'parsers': AIParserResource, 'flows': AIFlow, 'services': AIService,
|
||||
'tasks': AITask, 'artifacts': AIArtifact}
|
||||
|
||||
|
||||
def admin(user):
|
||||
from common.constants.permission_constants import RoleConstants
|
||||
return user.role == str(RoleConstants.ADMIN.value)
|
||||
|
||||
|
||||
def require_admin(user):
|
||||
if not admin(user):
|
||||
raise PermissionDenied('仅管理员可操作')
|
||||
|
||||
|
||||
def visible(query, user):
|
||||
# Sharing is explicit, with no implicit public access.
|
||||
if admin(user):
|
||||
return query
|
||||
if query.model in (AIParserResource, AIService):
|
||||
ids = [obj.id for obj in query if str(obj.owner_id) == str(user.id)
|
||||
or str(user.id) in obj.allowed_users]
|
||||
return query.filter(id__in=ids)
|
||||
return query.filter(owner_id=user.id)
|
||||
|
||||
|
||||
def one(kind, pk, workspace, user, edit=False):
|
||||
from rest_framework.serializers import UUIDField
|
||||
pk = UUIDField().run_validation(pk)
|
||||
model = KINDS.get(kind)
|
||||
if not model:
|
||||
raise NotFound()
|
||||
obj = visible(model.objects.filter(workspace_id=workspace), user).filter(id=pk).first()
|
||||
if not obj:
|
||||
raise NotFound('记录不存在或无权访问')
|
||||
if edit and not admin(user) and str(obj.owner_id) != str(user.id):
|
||||
raise PermissionDenied('无权修改')
|
||||
return obj
|
||||
|
||||
|
||||
def text(value, label, required=True, limit=MAX_TEXT):
|
||||
if not isinstance(value, str) or len(value) > limit or (required and not value.strip()):
|
||||
raise ValidationError(f'{label}无效')
|
||||
return value
|
||||
|
||||
|
||||
def integer(value, minimum, maximum, label):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
|
||||
raise ValidationError(f'{label}无效')
|
||||
return value
|
||||
|
||||
|
||||
def users(value):
|
||||
from users.models import User
|
||||
if not isinstance(value, list) or len(value) > 500:
|
||||
raise ValidationError('授权用户无效')
|
||||
values = {str(item) for item in value}
|
||||
try:
|
||||
allowed = set(str(pk) for pk in User.objects.filter(id__in=values, is_active=True).values_list('id', flat=True))
|
||||
except (ValueError, TypeError):
|
||||
raise ValidationError('授权用户无效')
|
||||
# Registration approval is enforced by the existing activation extension.
|
||||
from users.models.user_manage_ext import UserActivationApplication
|
||||
pending = set(str(pk) for pk in UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True))
|
||||
if values - (allowed - pending):
|
||||
raise ValidationError('只能授权已激活且启用的用户')
|
||||
return sorted(values)
|
||||
|
||||
|
||||
def authorized_models(workspace, user):
|
||||
from models_provider.models import Model
|
||||
from system_manage.models import WorkspaceUserResourcePermission, AuthTargetType
|
||||
query = Model.objects.filter(workspace_id=workspace, status='SUCCESS')
|
||||
if not admin(user):
|
||||
grants = WorkspaceUserResourcePermission.objects.filter(
|
||||
workspace_id=workspace, user_id=user.id, auth_target_type=AuthTargetType.MODEL)
|
||||
ids = [g.target for g in grants if 'VIEW' in g.permission_list]
|
||||
query = query.filter(id__in=ids)
|
||||
return query
|
||||
|
||||
|
||||
def model_for(workspace, user, model_id):
|
||||
from rest_framework.serializers import UUIDField
|
||||
model_id = UUIDField().run_validation(model_id)
|
||||
obj = authorized_models(workspace, user).filter(id=model_id, model_type='LLM').first()
|
||||
if not obj:
|
||||
raise PermissionDenied('模型不可用或未授权')
|
||||
return obj
|
||||
|
||||
|
||||
def serialize(obj, detail=False):
|
||||
data = {'id': str(obj.id), 'name': obj.name, 'updated_at': obj.updated_at.isoformat()}
|
||||
if isinstance(obj, AIParserResource):
|
||||
data.update(enabled=obj.enabled, allowed_users=obj.allowed_users,
|
||||
options=obj.options, credential_configured=bool(obj.credential))
|
||||
# Only callers of admin management endpoints receive endpoint addresses.
|
||||
elif isinstance(obj, AIFlow):
|
||||
data.update(revision=obj.revision, nodes=obj.nodes)
|
||||
elif isinstance(obj, AIService):
|
||||
data.update(flow_id=str(obj.flow_id), enabled=obj.enabled, allowed_users=obj.allowed_users,
|
||||
requests_per_minute=obj.requests_per_minute,
|
||||
key_configured=bool(obj.key_hash), url=f'ai_hub/published/{obj.id}/run')
|
||||
elif isinstance(obj, AITask):
|
||||
from system_manage.ai_hub_progress import task_progress
|
||||
data.update(status=obj.status, steps=obj.steps, error=obj.error, attempt=obj.attempt,
|
||||
started_at=obj.started_at, finished_at=obj.finished_at)
|
||||
if detail:
|
||||
data.update(snapshot=obj.snapshot, result=obj.result, progress=task_progress(obj))
|
||||
elif isinstance(obj, AIArtifact):
|
||||
data.update(revision=obj.revision, layout=obj.layout, task_id=str(obj.task_id) if obj.task_id else None)
|
||||
if detail:
|
||||
data.update(original=obj.original, markdown=obj.markdown, history=obj.history)
|
||||
return data
|
||||
|
||||
|
||||
def parser_save(workspace, user, data, obj=None):
|
||||
require_admin(user)
|
||||
endpoint = text(data.get('endpoint', getattr(obj, 'endpoint', '')), '地址', limit=1024).rstrip('/')
|
||||
parsed = urlsplit(endpoint)
|
||||
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValidationError('解析服务地址无效')
|
||||
obj = obj or AIParserResource(workspace_id=workspace, owner_id=user.id)
|
||||
obj.name = text(data.get('name', obj.name), '名称', limit=128)
|
||||
obj.endpoint = endpoint
|
||||
if 'api_key' in data:
|
||||
from common.utils.rsa_util import rsa_long_encrypt
|
||||
key = text(data['api_key'], '密钥', required=False, limit=4096)
|
||||
obj.credential = rsa_long_encrypt(key) if key else ''
|
||||
enabled = data.get('enabled', obj.enabled)
|
||||
if not isinstance(enabled, bool):
|
||||
raise ValidationError('启用状态无效')
|
||||
obj.enabled = enabled
|
||||
if 'allowed_users' in data:
|
||||
obj.allowed_users = users(data['allowed_users'])
|
||||
options = data.get('options', obj.options)
|
||||
if not isinstance(options, dict) or set(options) - {'backend', 'language', 'ocr', 'formula', 'table', 'protocol', 'tier'}:
|
||||
raise ValidationError('解析参数无效')
|
||||
if options.get('protocol', 'legacy') not in ('legacy', 'v1') or options.get('tier', 'standard') not in ('flash', 'basic', 'standard', 'advanced'):
|
||||
raise ValidationError('解析接口或档位无效')
|
||||
if options.get('backend', 'pipeline') not in ('pipeline', 'hybrid-auto-engine', 'vlm-auto-engine'):
|
||||
raise ValidationError('不支持的解析后端')
|
||||
if options.get('language', 'ch') not in ('ch', 'en', 'japan', 'korean', 'chinese_cht'):
|
||||
raise ValidationError('识别语言无效')
|
||||
if any(not isinstance(options[k], bool) for k in ('ocr', 'formula', 'table') if k in options):
|
||||
raise ValidationError('解析参数无效')
|
||||
obj.options = options
|
||||
obj.save()
|
||||
return obj
|
||||
|
||||
|
||||
def parser_test(parser):
|
||||
"""Probe the registered service API, without submitting files or returning service content."""
|
||||
import time
|
||||
import requests
|
||||
from urllib.parse import urlunsplit
|
||||
from common.utils.rsa_util import rsa_long_decrypt
|
||||
parsed = urlsplit(parser.endpoint.rstrip('/'))
|
||||
if parsed.scheme not in ('http', 'https') or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValidationError('解析服务地址无效')
|
||||
base_path = parsed.path
|
||||
for suffix in ('/v1', '/api'):
|
||||
if base_path.endswith(suffix):
|
||||
base_path = base_path[:-len(suffix)]
|
||||
break
|
||||
url = urlunsplit((parsed.scheme, parsed.netloc, base_path + '/openapi.json', '', ''))
|
||||
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=(3, 7), allow_redirects=False)
|
||||
if response.status_code != 200:
|
||||
raise ValidationError(f'连接测试失败(HTTP {response.status_code})')
|
||||
data = response.json()
|
||||
except requests.RequestException as exc:
|
||||
raise ValidationError('解析服务连接失败或超时') from exc
|
||||
except ValueError as exc:
|
||||
raise ValidationError('解析服务未返回有效接口定义') from exc
|
||||
paths = data.get('paths') if isinstance(data, dict) else None
|
||||
v1 = parser.options.get('protocol') == 'v1' or parsed.path.endswith('/v1')
|
||||
suffix = '/tasks' if v1 else '/file_parse'
|
||||
if not isinstance(paths, dict) or not any(path.endswith(suffix) and 'post' in methods
|
||||
for path, methods in paths.items() if isinstance(methods, dict)):
|
||||
raise ValidationError('未识别到对应的解析接口')
|
||||
return {'connected': True, 'elapsed_ms': round((time.monotonic() - started) * 1000),
|
||||
'message': '解析接口连接正常'}
|
||||
|
||||
|
||||
def validate_nodes(workspace, user, nodes):
|
||||
if not isinstance(nodes, list) or not 1 <= len(nodes) <= 3:
|
||||
raise ValidationError('请选择处理节点')
|
||||
types = [n.get('type') for n in nodes if isinstance(n, dict)]
|
||||
if len(types) != len(nodes) or types[-1] != 'render' or len(set(types)) != len(types):
|
||||
raise ValidationError('流程须以导图生成为终点,节点不可重复')
|
||||
if types != [t for t in ('parse', 'organize', 'render') if t in types]:
|
||||
raise ValidationError('节点顺序应为解析、整理、导图生成')
|
||||
result = []
|
||||
for node in nodes:
|
||||
kind = node['type']
|
||||
clean = {'type': kind}
|
||||
if kind == 'parse':
|
||||
parser = one('parsers', node.get('resource_id'), workspace, user)
|
||||
if not parser.enabled:
|
||||
raise ValidationError('解析服务已停用')
|
||||
clean['resource_id'] = str(parser.id)
|
||||
if kind == 'organize':
|
||||
clean['model_id'] = str(model_for(workspace, user, node.get('model_id')).id)
|
||||
clean['prompt'] = text(node.get('prompt', '保留事实、数值和原文语言,将以下文档整理成分级Markdown标题和要点。'), '提示词', limit=16000)
|
||||
temperature = node.get('temperature', 0.2)
|
||||
if isinstance(temperature, bool) or not isinstance(temperature, (int, float)) or not 0 <= temperature <= 2:
|
||||
raise ValidationError('温度无效')
|
||||
clean['temperature'] = temperature
|
||||
result.append(clean)
|
||||
return result
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def flow_create(workspace, user, data):
|
||||
nodes = validate_nodes(workspace, user, data.get('nodes'))
|
||||
# Every save is immutable; published services always reference a fixed row.
|
||||
previous = one('flows', data['previous_id'], workspace, user, edit=True) if data.get('previous_id') else None
|
||||
return AIFlow.objects.create(workspace_id=workspace, owner_id=user.id,
|
||||
name=text(data.get('name'), '名称', limit=128), nodes=nodes,
|
||||
revision=previous.revision + 1 if previous else 1)
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def service_create(workspace, user, data):
|
||||
require_admin(user)
|
||||
flow = one('flows', data.get('flow_id'), workspace, user)
|
||||
validate_nodes(workspace, user, flow.nodes)
|
||||
return AIService.objects.create(workspace_id=workspace, owner_id=user.id,
|
||||
name=text(data.get('name'), '名称', limit=128), flow=flow,
|
||||
allowed_users=users(data.get('allowed_users', [])),
|
||||
requests_per_minute=integer(data.get('requests_per_minute', 20), 1, 600, '调用频率'))
|
||||
|
||||
|
||||
def make_snapshot(workspace, user, nodes):
|
||||
clean = validate_nodes(workspace, user, nodes)
|
||||
for node in clean:
|
||||
if node['type'] == 'parse':
|
||||
parser = one('parsers', node['resource_id'], workspace, user)
|
||||
node['options'] = dict(parser.options)
|
||||
node['resource_version'] = parser.updated_at.isoformat()
|
||||
if node['type'] == 'organize':
|
||||
model = model_for(workspace, user, node['model_id'])
|
||||
node.update(model_name=model.model_name, provider=model.provider, resource_version=model.update_time.isoformat())
|
||||
return clean
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def submit(workspace, user, data, upload=None, service=None):
|
||||
from users.models import User
|
||||
current_user = User.objects.select_for_update().get(id=user.id)
|
||||
if not current_user.is_active:
|
||||
raise PermissionDenied('用户已停用')
|
||||
if AITask.objects.filter(owner_id=user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
|
||||
raise ValidationError('待处理任务已达上限')
|
||||
if service:
|
||||
if not service.enabled:
|
||||
raise ValidationError('服务已停用')
|
||||
nodes = service.flow.nodes
|
||||
else:
|
||||
nodes = one('flows', data['flow_id'], workspace, user).nodes if data.get('flow_id') else data.get('nodes')
|
||||
snapshot = make_snapshot(workspace, user, nodes)
|
||||
markdown = text(data.get('markdown', ''), '文档内容', required=False, limit=MAX_INLINE_TEXT)
|
||||
if upload and upload.size > MAX_FILE:
|
||||
raise ValidationError('文件不能超过20MB')
|
||||
if upload and snapshot[0]['type'] != 'parse':
|
||||
raise ValidationError('文件任务需要解析节点')
|
||||
if not upload and snapshot[0]['type'] == 'parse':
|
||||
raise ValidationError('请选择文件')
|
||||
if not upload and not markdown.strip():
|
||||
raise ValidationError('文档内容不能为空')
|
||||
from system_manage.ai_hub_assets import asset_lock, normalize
|
||||
with asset_lock():
|
||||
markdown = text(normalize(markdown, workspace, user), '文档内容', required=False)
|
||||
return AITask.objects.create(workspace_id=workspace, owner_id=user.id,
|
||||
name=text(data.get('name') or (upload.name if upload else '导图任务'), '名称', limit=128), service=service,
|
||||
snapshot={'nodes': snapshot, 'flow_id': str(service.flow_id) if service else data.get('flow_id'),
|
||||
'flow_revision': service.flow.revision if service else None},
|
||||
input_data={'markdown': markdown}, file_name=upload.name[:255] if upload else '',
|
||||
file_data=upload.read() if upload else None)
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def artifact_save(workspace, user, data, obj=None):
|
||||
if obj:
|
||||
obj = AIArtifact.objects.select_for_update().get(pk=obj.pk)
|
||||
if data.get('revision') != obj.revision:
|
||||
raise ValidationError('内容已更新,请重新打开')
|
||||
history = list(obj.history)
|
||||
if len(history) >= 30:
|
||||
raise ValidationError('修订已达30次,请另存为新图谱')
|
||||
history.append({'revision': obj.revision, 'markdown': obj.markdown, 'layout': obj.layout,
|
||||
'operator': str(user.id), 'updated_at': obj.updated_at.isoformat()})
|
||||
obj.history = history
|
||||
obj.revision += 1
|
||||
else:
|
||||
task = one('tasks', data['task_id'], workspace, user) if data.get('task_id') else None
|
||||
if task and task.status != 'SUCCESS':
|
||||
raise ValidationError('任务尚未成功')
|
||||
obj = AIArtifact(workspace_id=workspace, owner_id=user.id, task=task,
|
||||
original=task.result.get('original', '') if task else text(data.get('original', ''), '原始内容', False, MAX_INLINE_TEXT))
|
||||
obj.name = text(data.get('name'), '名称', limit=128)
|
||||
obj.markdown = text(data.get('markdown'), '修订内容', limit=MAX_INLINE_TEXT)
|
||||
obj.layout = data.get('layout', 'tree')
|
||||
if obj.layout not in ('tree', 'radial', 'horizontal', 'vertical'):
|
||||
raise ValidationError('布局无效')
|
||||
from system_manage.ai_hub_assets import asset_lock, normalize
|
||||
with asset_lock():
|
||||
obj.markdown = text(normalize(obj.markdown, workspace, user), '修订内容')
|
||||
obj.original = text(normalize(obj.original, workspace, user), '原始内容', required=False)
|
||||
for entry in obj.history:
|
||||
entry['markdown'] = normalize(entry.get('markdown', ''), workspace, user)
|
||||
obj.save()
|
||||
return obj
|
||||
|
||||
|
||||
def parse_file(parser, name, content, options=None, progress_callback=None):
|
||||
import requests
|
||||
from common.utils.rsa_util import rsa_long_decrypt
|
||||
opts = options if options is not None else parser.options
|
||||
headers = {'Authorization': 'Bearer ' + rsa_long_decrypt(parser.credential)} if parser.credential else {}
|
||||
if opts.get('protocol') == 'v1' or parser.endpoint.rstrip('/').endswith('/v1'):
|
||||
from system_manage.ai_hub_parser_v1 import parse
|
||||
return parse(parser, name, content, opts, headers, progress_callback=progress_callback)
|
||||
try:
|
||||
response = requests.post(parser.endpoint + '/file_parse', files={'files': (name, content)}, headers=headers,
|
||||
data={'backend': opts.get('backend', 'pipeline'), 'lang_list': opts.get('language', 'ch'),
|
||||
'parse_method': 'ocr' if opts.get('ocr') else 'auto', 'return_md': 'true',
|
||||
'return_middle_json': 'true', 'response_format_zip': 'false', 'return_images': 'true',
|
||||
'formula_enable': str(opts.get('formula', True)).lower(),
|
||||
'table_enable': str(opts.get('table', True)).lower()}, timeout=(10, 300), allow_redirects=False)
|
||||
except Exception as exc:
|
||||
raise ValidationError('解析服务连接失败或超时') from exc
|
||||
if not 200 <= response.status_code < 300:
|
||||
raise ValidationError('解析服务调用失败')
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise ValidationError('解析服务返回格式无效') from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValidationError('解析服务返回格式无效')
|
||||
entries = payload.get('results', {})
|
||||
if not isinstance(entries, dict) or len(entries) != 1:
|
||||
raise ValidationError('解析结果格式无效')
|
||||
result = next(iter(entries.values()))
|
||||
if not isinstance(result, dict):
|
||||
raise ValidationError('解析结果格式无效')
|
||||
markdown = text(result.get('md_content'), '解析结果', limit=MAX_INLINE_TEXT)
|
||||
return {'markdown': markdown, 'source': result.get('middle_json', {}), 'parser_id': str(parser.id),
|
||||
'images': result.get('images', {})}
|
||||
|
||||
|
||||
def markdown_tree(markdown):
|
||||
"""Markdown转稳定ID结构树:标题层级、嵌套列表、表格;围栏内容不参与结构。"""
|
||||
import re
|
||||
root = {'id': 'root', 'text': '文档', 'children': []}
|
||||
heading_stack = [(0, root)]
|
||||
list_stack = []
|
||||
fence = None
|
||||
table_buffer, table_start = [], None
|
||||
|
||||
def container():
|
||||
return list_stack[-1][1] if list_stack else heading_stack[-1][1]
|
||||
|
||||
def flush_table():
|
||||
nonlocal table_buffer, table_start
|
||||
if not table_buffer:
|
||||
return
|
||||
index, lines = table_start, table_buffer
|
||||
table_buffer, table_start = [], None
|
||||
|
||||
def cells(raw):
|
||||
return [cell.strip() for cell in raw.strip().strip('|').split('|')]
|
||||
|
||||
if len(lines) >= 2 and re.match(r'^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$', lines[1]):
|
||||
headers = cells(lines[0])
|
||||
aligns = []
|
||||
for cell in cells(lines[1]):
|
||||
left, right = cell.startswith(':'), cell.endswith(':')
|
||||
aligns.append('center' if left and right else 'right' if right else 'left')
|
||||
rows = [cells(raw) for raw in lines[2:]]
|
||||
node = {'id': f'n{index}', 'type': 'table', 'text': '表格:' + '、'.join(headers),
|
||||
'table': {'headers': headers, 'aligns': aligns, 'rows': rows,
|
||||
'row_count': len(rows), 'column_count': len(headers)},
|
||||
'children': []}
|
||||
container()['children'].append(node)
|
||||
else:
|
||||
for offset, raw in enumerate(lines):
|
||||
container()['children'].append({'id': f'n{index + offset}', 'text': raw.strip(), 'children': []})
|
||||
|
||||
list_pattern = re.compile(r'^(\s*)(?:([-*+])|(\d{1,9})[.)])\s+(.+)$')
|
||||
for index, line in enumerate(markdown.splitlines()):
|
||||
fence_match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
|
||||
if fence_match:
|
||||
token = fence_match.group(1)
|
||||
if fence is None:
|
||||
fence = token
|
||||
flush_table()
|
||||
list_stack.clear()
|
||||
elif token[0] == fence[0] and len(token) >= len(fence):
|
||||
fence = None
|
||||
continue
|
||||
if fence:
|
||||
continue
|
||||
stripped = line.strip()
|
||||
if stripped.startswith('|') and stripped.count('|') >= 2:
|
||||
if table_start is None:
|
||||
table_start = index
|
||||
list_stack.clear()
|
||||
table_buffer.append(line)
|
||||
continue
|
||||
flush_table()
|
||||
if not stripped:
|
||||
continue
|
||||
heading = re.match(r'^\s{0,3}(#{1,6})\s+(.+)', line)
|
||||
if heading:
|
||||
level = len(heading.group(1))
|
||||
list_stack.clear()
|
||||
while heading_stack[-1][0] >= level:
|
||||
heading_stack.pop()
|
||||
node = {'id': f'n{index}', 'text': heading.group(2).rstrip('# ').strip(), 'children': []}
|
||||
heading_stack[-1][1]['children'].append(node)
|
||||
heading_stack.append((level, node))
|
||||
continue
|
||||
item = list_pattern.match(line)
|
||||
if item:
|
||||
indent = len(item.group(1).expandtabs(4))
|
||||
while list_stack and list_stack[-1][0] >= indent:
|
||||
list_stack.pop()
|
||||
ordered = item.group(3)
|
||||
node = {'id': f'n{index}', 'type': 'list-item',
|
||||
'text': item.group(4).strip(),
|
||||
'list': {'type': 'ol' if ordered else 'ul', 'index': int(ordered) if ordered else None},
|
||||
'children': []}
|
||||
container()['children'].append(node)
|
||||
list_stack.append((indent, node))
|
||||
continue
|
||||
if line[:1].isspace() and list_stack:
|
||||
list_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
|
||||
continue
|
||||
list_stack.clear()
|
||||
heading_stack[-1][1]['children'].append({'id': f'n{index}', 'text': stripped, 'children': []})
|
||||
flush_table()
|
||||
return root
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
"""Private, durable image files. References live in existing task/artifact records.
|
||||
|
||||
AI_HUB_ASSET_DIR must be on persistent shared storage when Web/worker are on
|
||||
different hosts. No migration, public static route, or Redis image payloads.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from urllib.parse import quote
|
||||
|
||||
from django.conf import settings
|
||||
from rest_framework.exceptions import NotFound, ValidationError
|
||||
|
||||
DATA_IMAGE = re.compile(r'data:image/(?:png|jpe?g|gif|webp|bmp);base64,[A-Za-z0-9+/=\r\n]+', re.I)
|
||||
ASSET = re.compile(r'/admin/api/workspace/([^/\s"<>]+)/ai_hub/assets/([a-f0-9-]{36})/content')
|
||||
MAX_IMAGE = 20 * 1024 * 1024
|
||||
MAX_IMAGES_BYTES = 100 * 1024 * 1024
|
||||
TTL = 7 * 86400
|
||||
|
||||
|
||||
def root():
|
||||
return Path(os.environ.get('AI_HUB_ASSET_DIR') or getattr(settings, 'AI_HUB_ASSET_DIR', '')
|
||||
or Path(__file__).resolve().parents[2] / 'data' / 'ai-hub-assets').resolve()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def asset_lock():
|
||||
directory = root()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
lock = directory / '.lock'
|
||||
stream = lock.open('a+b')
|
||||
stream.seek(0); stream.write(b'0'); stream.flush(); stream.seek(0)
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
if os.name == 'nt':
|
||||
import msvcrt
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
break
|
||||
except OSError:
|
||||
if time.monotonic() >= deadline:
|
||||
stream.close()
|
||||
raise ValidationError('图片存储繁忙,请稍后重试')
|
||||
time.sleep(.05)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if os.name == 'nt':
|
||||
stream.seek(0)
|
||||
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
||||
stream.close()
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
temporary = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
|
||||
try:
|
||||
with temporary.open('wb') as stream:
|
||||
stream.write(content)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _metadata(asset_id):
|
||||
try:
|
||||
asset_id = str(uuid.UUID(str(asset_id)))
|
||||
return json.loads((root() / (asset_id + '.json')).read_text(encoding='utf-8'))
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise NotFound('图片不存在或已过期')
|
||||
|
||||
|
||||
def authorize(workspace, user, asset_id):
|
||||
from system_manage.ai_hub import admin
|
||||
meta = _metadata(asset_id)
|
||||
if meta['workspace_id'] != workspace or (meta['owner_id'] != str(user.id) and not admin(user)):
|
||||
raise NotFound('图片不存在或无权访问')
|
||||
return meta
|
||||
|
||||
|
||||
def image_bytes(workspace, user, asset_id):
|
||||
meta = authorize(workspace, user, asset_id)
|
||||
try:
|
||||
content = (root() / (str(uuid.UUID(str(asset_id))) + '.bin')).read_bytes()
|
||||
except OSError:
|
||||
raise NotFound('图片不存在或已过期')
|
||||
return content, meta
|
||||
|
||||
|
||||
def _store(uri, workspace, user):
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
try:
|
||||
encoded = uri.split(',', 1)[1]
|
||||
if len(encoded) > MAX_IMAGE * 4 // 3 + 16:
|
||||
raise ValueError()
|
||||
content = base64.b64decode(re.sub(r'\s', '', encoded), validate=True)
|
||||
if not content or len(content) > MAX_IMAGE:
|
||||
raise ValueError()
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise ValueError()
|
||||
fmt = image.format
|
||||
image.verify()
|
||||
mime = {'PNG': 'image/png', 'JPEG': 'image/jpeg', 'GIF': 'image/gif',
|
||||
'WEBP': 'image/webp', 'BMP': 'image/bmp'}[fmt]
|
||||
except (ValueError, KeyError, OSError, UnidentifiedImageError, Image.DecompressionBombError):
|
||||
raise ValidationError('图片格式无效或超过大小限制')
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
asset_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f'ai-hub:{workspace}:{user.id}:{digest}'))
|
||||
meta = {'id': asset_id, 'workspace_id': workspace, 'owner_id': str(user.id),
|
||||
'sha256': digest, 'mime': mime, 'size': len(content), 'touched_at': time.time()}
|
||||
if not (root() / (asset_id + '.bin')).exists():
|
||||
_write(root() / (asset_id + '.bin'), content)
|
||||
_write(root() / (asset_id + '.json'), json.dumps(meta).encode())
|
||||
return f'/admin/api/workspace/{quote(workspace, safe="")}/ai_hub/assets/{asset_id}/content'
|
||||
|
||||
|
||||
def normalize(markdown, workspace, user, images=None):
|
||||
"""Caller holds asset_lock until its database reference is saved."""
|
||||
replacements = {}
|
||||
total = 0
|
||||
|
||||
def store(uri):
|
||||
nonlocal total
|
||||
if uri not in replacements:
|
||||
total += len(uri) * 3 // 4
|
||||
if total > MAX_IMAGES_BYTES:
|
||||
raise ValidationError('文档图片总大小超过限制')
|
||||
replacements[uri] = _store(uri, workspace, user)
|
||||
return replacements[uri]
|
||||
|
||||
if images:
|
||||
if not isinstance(images, dict):
|
||||
raise ValidationError('解析图片格式无效')
|
||||
# Only replace destinations in Markdown/HTML, never arbitrary words.
|
||||
for name, uri in images.items():
|
||||
if not isinstance(name, str) or not isinstance(uri, str) or not DATA_IMAGE.fullmatch(uri):
|
||||
continue
|
||||
pattern = re.compile(r'(?P<prefix>\]\(<?|src=["\x27]|^\s*\[[^\]\n]+\]:\s*<?)(?:\./)?(?:images/)?'
|
||||
+ re.escape(name.removeprefix('images/')) + r'(?=[\s)>"\x27])', re.M)
|
||||
if pattern.search(markdown):
|
||||
url = store(uri)
|
||||
markdown = pattern.sub(lambda m: m['prefix'] + url, markdown)
|
||||
markdown = DATA_IMAGE.sub(lambda m: store(m[0]), markdown)
|
||||
for match in ASSET.finditer(markdown):
|
||||
if match[1] != quote(workspace, safe=''):
|
||||
raise ValidationError('不能引用其他工作空间的图片')
|
||||
meta = authorize(workspace, user, match[2])
|
||||
meta['touched_at'] = time.time()
|
||||
_write(root() / (match[2] + '.json'), json.dumps(meta).encode())
|
||||
return markdown
|
||||
|
||||
|
||||
def protect_images(markdown):
|
||||
"""Keep image destinations out of model rewriting; reject lost/forged tokens."""
|
||||
images = []
|
||||
pattern = re.compile(r'!\[[^\]\n]*\]\([^\n]*?\)|<img\b[^>]*>', re.I)
|
||||
def replace(match):
|
||||
images.append(match[0])
|
||||
return f'AIHUBIMAGE{len(images):04d}TOKEN'
|
||||
return pattern.sub(replace, markdown), images
|
||||
|
||||
|
||||
def restore_images(markdown, images):
|
||||
# Summarizers may omit or repeat placeholders. The backend owns the image
|
||||
# manifest, so preserve known positions and append omitted originals once.
|
||||
known = {str(index + 1): image for index, image in enumerate(images)}
|
||||
restored = set()
|
||||
def replace(match):
|
||||
index = match[1].lstrip('0') or '0'
|
||||
if index not in known or index in restored:
|
||||
return ''
|
||||
restored.add(index)
|
||||
return known[index]
|
||||
markdown = re.sub(r'AIHUBIMAGE[\s_*`-]*(\d+)[\s_*`-]*TOKEN', replace, markdown, flags=re.I)
|
||||
missing = [image for index, image in known.items() if index not in restored]
|
||||
if missing:
|
||||
markdown = markdown.rstrip() + '\n\n## 原文图片\n\n' + '\n\n'.join(missing)
|
||||
return markdown
|
||||
|
||||
|
||||
def cleanup(apply=False, now=None):
|
||||
"""Only expired, unreferenced Hub images; saved originals/history count as refs."""
|
||||
from system_manage.models.ai_hub import AIArtifact, AITask
|
||||
cutoff = (now or time.time()) - TTL
|
||||
expired = []
|
||||
with asset_lock():
|
||||
protected = set()
|
||||
for artifact in AIArtifact.objects.only('markdown', 'original', 'history').iterator():
|
||||
content = artifact.markdown + '\n' + artifact.original + '\n' + json.dumps(artifact.history)
|
||||
protected.update(m[2] for m in ASSET.finditer(content))
|
||||
for task in AITask.objects.filter(status__in=['PENDING', 'RUNNING']).only('input_data', 'result').iterator():
|
||||
protected.update(m[2] for m in ASSET.finditer(json.dumps([task.input_data, task.result])))
|
||||
for path in root().glob('*.json'):
|
||||
meta = _metadata(path.stem)
|
||||
if meta['touched_at'] >= cutoff or path.stem in protected:
|
||||
continue
|
||||
expired.append(path.stem)
|
||||
if apply:
|
||||
(root() / (path.stem + '.bin')).unlink(missing_ok=True)
|
||||
path.unlink()
|
||||
return len(expired)
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
"""Knowledge import uses the same authorized parser and durable task result as mindmaps."""
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from rest_framework.exceptions import PermissionDenied, ValidationError
|
||||
from knowledge.models import Knowledge
|
||||
from system_manage import ai_hub as hub
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def parser_binding(workspace, knowledge_id, user, data=None):
|
||||
knowledge = Knowledge.objects.select_for_update().filter(id=knowledge_id, workspace_id=workspace).first()
|
||||
if not knowledge:
|
||||
raise ValidationError('知识库不存在')
|
||||
meta = dict(knowledge.meta or {})
|
||||
if data is not None:
|
||||
parser_id = data.get('parser_id')
|
||||
if parser_id:
|
||||
parser = hub.one('parsers', parser_id, workspace, user)
|
||||
if not parser.enabled:
|
||||
raise ValidationError('解析服务已停用')
|
||||
meta['parser_id'] = str(parser_id) if parser_id else ''
|
||||
knowledge.meta = meta
|
||||
knowledge.save(update_fields=['meta', 'update_time'])
|
||||
return {'parser_id': meta.get('parser_id', '')}
|
||||
|
||||
|
||||
def split_with_parser(workspace, knowledge_id, user, file, patterns, with_filter, limit, overlap):
|
||||
knowledge = Knowledge.objects.get(id=knowledge_id, workspace_id=workspace)
|
||||
parser_id = (knowledge.meta or {}).get('parser_id')
|
||||
if not parser_id:
|
||||
return None
|
||||
if user is None:
|
||||
raise PermissionDenied('解析服务调用需要用户身份')
|
||||
from system_manage.ai_hub_worker import execute
|
||||
# Mark as running before commit: the asynchronous consumer cannot claim the same task.
|
||||
with transaction.atomic():
|
||||
task = hub.submit(workspace, user, {'name': file.name[:128], 'nodes': [
|
||||
{'type': 'parse', 'resource_id': parser_id}, {'type': 'render'}]}, upload=file)
|
||||
task.status = 'RUNNING'
|
||||
task.started_at = timezone.now()
|
||||
task.save(update_fields=['status', 'started_at'])
|
||||
execute(task)
|
||||
file.seek(0)
|
||||
if task.status != 'SUCCESS':
|
||||
raise ValidationError(task.error or '解析失败')
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from common.handle.impl.text.text_split_handle import TextSplitHandle
|
||||
content = task.result['markdown'].encode('utf-8')
|
||||
converted = SimpleUploadedFile(file.name + '.md', content)
|
||||
result = TextSplitHandle().handle(converted, patterns, with_filter, limit,
|
||||
lambda _: content, lambda _: None, overlap)
|
||||
result['name'] = file.name
|
||||
return result, {'resource_id': str(parser_id), 'task_id': str(task.id),
|
||||
'resource_version': task.snapshot['nodes'][0]['resource_version']}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
"""MinerU v1 task API adapter. Only follows paths built from the registered endpoint."""
|
||||
import re
|
||||
import time
|
||||
import requests
|
||||
from rest_framework.exceptions import ValidationError
|
||||
|
||||
|
||||
def _json(response):
|
||||
if response.status_code not in (200, 201, 202):
|
||||
raise ValidationError(f'解析服务调用失败(HTTP {response.status_code})')
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as exc:
|
||||
raise ValidationError('解析服务返回格式无效') from exc
|
||||
if not isinstance(body, dict):
|
||||
raise ValidationError('解析服务返回格式无效')
|
||||
return body
|
||||
|
||||
|
||||
def parse(parser, name, content, options, headers, progress_callback=None):
|
||||
from system_manage.ai_hub import text, MAX_INLINE_TEXT
|
||||
endpoint = parser.endpoint.rstrip('/')
|
||||
deadline = time.monotonic() + 300
|
||||
try:
|
||||
response = requests.post(endpoint + '/tasks', files={'files': (name, content)}, headers=headers,
|
||||
params=[('tier', options.get('tier', 'standard')), ('ocr_mode', 'ocr' if options.get('ocr') else 'auto'),
|
||||
('output_formats', 'markdown'), ('output_formats', 'middle_json')],
|
||||
timeout=(10, 30), allow_redirects=False)
|
||||
task = _json(response)
|
||||
from system_manage.ai_hub_progress import parser_progress
|
||||
task_id = task.get('task_id', '')
|
||||
if not isinstance(task_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', task_id):
|
||||
raise ValidationError('解析服务未返回有效任务ID')
|
||||
if progress_callback:
|
||||
progress_callback(parser_progress(task))
|
||||
while task.get('status') in ('queued', 'running'):
|
||||
if time.monotonic() >= deadline:
|
||||
raise ValidationError('解析等待超时,后台任务ID:' + task_id)
|
||||
time.sleep(2)
|
||||
task = _json(requests.get(endpoint + '/tasks/' + task_id, headers=headers,
|
||||
timeout=(10, 20), allow_redirects=False))
|
||||
if progress_callback:
|
||||
progress_callback(parser_progress(task))
|
||||
if task.get('status') != 'completed':
|
||||
raise ValidationError('解析任务未成功,后台任务ID:' + task_id)
|
||||
result = _json(requests.get(endpoint + '/tasks/' + task_id + '/result', headers=headers,
|
||||
timeout=(10, 30), allow_redirects=False))
|
||||
except requests.RequestException as exc:
|
||||
raise ValidationError('解析服务连接失败或超时') from exc
|
||||
files = result.get('files', [])
|
||||
if not isinstance(files, list) or len(files) != 1 or not isinstance(files[0], dict) or files[0].get('status') != 'completed':
|
||||
raise ValidationError('解析文件结果无效')
|
||||
output = files[0].get('content', {})
|
||||
if not isinstance(output, dict):
|
||||
raise ValidationError('解析文件内容无效')
|
||||
markdown = text(output.get('markdown'), '解析结果', limit=MAX_INLINE_TEXT)
|
||||
return {'markdown': markdown, 'parser_id': str(parser.id), 'images': output.get('images', {}),
|
||||
'source': {'task_id': task_id, 'parse': files[0].get('parse'),
|
||||
'middle_json': output.get('middle_json', {})}}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
"""Expose real phases/counts only. A single-file v1 job has no page percentage."""
|
||||
from django.utils import timezone
|
||||
|
||||
LABELS = {'parse': '文档解析中', 'organize': '智能整理中', 'render': '生成思维导图'}
|
||||
|
||||
|
||||
def parser_progress(task):
|
||||
state = task.get('status')
|
||||
phase = {'queued': '等待解析服务处理', 'running': '正在解析文档', 'completed': '获取解析结果'}.get(state, '正在解析文档')
|
||||
raw = task.get('progress') or {}
|
||||
def count(key):
|
||||
value = raw.get(key, 0) if isinstance(raw, dict) else 0
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
|
||||
completed, total = count('completed'), count('total')
|
||||
percent = min(100, completed * 100 // total) if total > 1 and completed else None
|
||||
if total > 1:
|
||||
phase += f'({min(completed, total)}/{total} 个文件)'
|
||||
return {'stage': phase, 'percent': percent, 'completed_files': completed, 'total_files': total}
|
||||
|
||||
|
||||
def task_progress(task):
|
||||
elapsed = max(0, int(((task.finished_at or timezone.now()) - (task.started_at or task.created_at)).total_seconds()))
|
||||
if task.status == 'SUCCESS':
|
||||
return {'stage': '完成', 'percent': 100, 'elapsed_seconds': elapsed}
|
||||
if task.status == 'PENDING':
|
||||
return {'stage': '等待任务处理', 'percent': None, 'elapsed_seconds': elapsed}
|
||||
if task.status == 'FAILED':
|
||||
return {'stage': '处理失败', 'percent': None, 'elapsed_seconds': elapsed}
|
||||
current = next((step for step in task.steps if step.get('status') == 'RUNNING'), {})
|
||||
detail = current.get('progress') or {}
|
||||
percent = detail.get('percent')
|
||||
total_steps = max(1, len(task.snapshot.get('nodes', [])))
|
||||
complete_steps = sum(step.get('status') == 'SUCCESS' for step in task.steps)
|
||||
if percent is not None:
|
||||
percent = min(99, int((complete_steps * 100 + percent) / total_steps))
|
||||
return {'stage': detail.get('stage') or LABELS.get(current.get('type'), '处理中'),
|
||||
'percent': percent, 'elapsed_seconds': elapsed}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
"""Database-backed single consumer; never consumes legacy Celery queues."""
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from system_manage.models.ai_hub import AITask
|
||||
from system_manage import ai_hub as hub
|
||||
|
||||
|
||||
def claim():
|
||||
with transaction.atomic():
|
||||
task = AITask.objects.select_for_update(skip_locked=True).filter(status='PENDING').order_by('created_at').first()
|
||||
if not task:
|
||||
return None
|
||||
task.status = 'RUNNING'
|
||||
task.started_at = timezone.now()
|
||||
task.save(update_fields=['status', 'started_at', 'updated_at'])
|
||||
return task
|
||||
|
||||
|
||||
def execute(task):
|
||||
from users.models import User
|
||||
steps = []
|
||||
try:
|
||||
user = User.objects.get(pk=task.owner_id)
|
||||
if not user.is_active:
|
||||
raise ValueError('用户已停用')
|
||||
if task.service_id:
|
||||
service = hub.one('services', task.service_id, task.workspace_id, user)
|
||||
if not service.enabled:
|
||||
raise ValueError('服务已停用')
|
||||
markdown = task.input_data.get('markdown', '')
|
||||
original = markdown
|
||||
source = {}
|
||||
tree = {}
|
||||
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree}
|
||||
task.save(update_fields=['result', 'updated_at'])
|
||||
for node in task.snapshot['nodes']:
|
||||
step = {'type': node['type'], 'status': 'RUNNING'}
|
||||
steps.append(step)
|
||||
task.steps = steps
|
||||
task.save(update_fields=['steps', 'updated_at'])
|
||||
if node['type'] == 'parse':
|
||||
parser = hub.one('parsers', node['resource_id'], task.workspace_id, user)
|
||||
if not parser.enabled or parser.updated_at.isoformat() != node['resource_version']:
|
||||
raise ValueError('解析服务已停用或版本发生变化,请重新提交')
|
||||
def report_progress(progress):
|
||||
step['progress'] = progress
|
||||
AITask.objects.filter(pk=task.pk, attempt=task.attempt, status='RUNNING').update(
|
||||
steps=steps, updated_at=timezone.now())
|
||||
report_progress({'stage': '上传文档到解析服务', 'percent': None})
|
||||
parsed = hub.parse_file(parser, task.file_name, bytes(task.file_data), node['options'], progress_callback=report_progress)
|
||||
from system_manage.ai_hub_assets import asset_lock, normalize
|
||||
with asset_lock():
|
||||
markdown = original = hub.text(normalize(parsed['markdown'], task.workspace_id, user, parsed.get('images')), '解析结果')
|
||||
source = parsed['source']
|
||||
# Source JSON can contain image data too; preserve structure without inline payloads.
|
||||
import json
|
||||
source = json.loads(normalize(json.dumps(source), task.workspace_id, user))
|
||||
task.result = {'original': original, 'markdown': markdown, 'source': source}
|
||||
task.save(update_fields=['result', 'updated_at'])
|
||||
elif node['type'] == 'organize':
|
||||
from models_provider.tools import get_model, get_model_default_params
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
model = hub.model_for(task.workspace_id, user, node['model_id'])
|
||||
if model.update_time.isoformat() != node['resource_version']:
|
||||
raise ValueError('模型配置已变更,请重新提交')
|
||||
# Bound input; never silently truncate the original document.
|
||||
from system_manage.ai_hub_assets import protect_images, restore_images
|
||||
model_markdown, image_tokens = protect_images(markdown)
|
||||
if len(model_markdown) > 60000:
|
||||
raise ValueError('文档超过智能整理长度限制,请分批处理')
|
||||
try:
|
||||
instance = get_model(model, **{**get_model_default_params(model),
|
||||
'temperature': node.get('temperature', 0.2), 'timeout': 180})
|
||||
prompt = node['prompt'] + '\n保留 AIHUBIMAGE 数字 TOKEN 图片占位符,尽量放在相关内容旁;不要编造图片或重复占位符。'
|
||||
response = instance.invoke([SystemMessage(content=prompt), HumanMessage(content=model_markdown)])
|
||||
except Exception as exc:
|
||||
from rest_framework.exceptions import ValidationError
|
||||
raise ValidationError('模型调用失败,请检查接入状态或输入长度') from exc
|
||||
markdown = restore_images(hub.text(response.content, '整理结果'), image_tokens)
|
||||
else:
|
||||
tree = hub.markdown_tree(markdown)
|
||||
step['status'] = 'SUCCESS'
|
||||
task.steps = steps
|
||||
# Persist intermediate original output so a later model failure does not lose OCR work.
|
||||
task.result = {'original': original, 'markdown': markdown, 'source': source, 'tree': tree}
|
||||
task.save(update_fields=['steps', 'result', 'updated_at'])
|
||||
task.status = 'SUCCESS'
|
||||
except Exception as exc:
|
||||
if steps:
|
||||
steps[-1]['status'] = 'FAILED'
|
||||
# Do not echo URLs, credentials, request bodies or provider exceptions to users/logs.
|
||||
from rest_framework.exceptions import APIException
|
||||
if isinstance(exc, APIException):
|
||||
detail = exc.detail
|
||||
while isinstance(detail, (list, tuple, dict)) and detail:
|
||||
detail = next(iter(detail.values())) if isinstance(detail, dict) else detail[0]
|
||||
task.error = str(detail)[:512]
|
||||
else:
|
||||
task.error = str(exc)[:512] if isinstance(exc, ValueError) else '执行失败,请检查服务状态后重试'
|
||||
task.status = 'FAILED'
|
||||
task.steps = steps
|
||||
task.finished_at = timezone.now()
|
||||
task.save(update_fields=['status', 'error', 'steps', 'finished_at', 'updated_at'])
|
||||
return task
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from django.core.management.base import BaseCommand
|
||||
from system_manage.ai_hub_assets import cleanup
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Check expired Hub images; --apply removes only images not referenced by saved graphs/history.'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--apply', action='store_true')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
count = cleanup(apply=options['apply'])
|
||||
self.stdout.write(f'{"Removed" if options["apply"] else "Eligible"}: {count}')
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import time
|
||||
from datetime import timedelta
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import close_old_connections
|
||||
from django.utils import timezone
|
||||
from system_manage.models.ai_hub import AITask
|
||||
from system_manage.ai_hub_worker import claim, execute
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Run the isolated AI hub task consumer (does not run migrations).'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--once', action='store_true')
|
||||
parser.add_argument('--recover', action='store_true', help='Mark running tasks older than 30 minutes failed')
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options['recover']:
|
||||
AITask.objects.filter(status='RUNNING', updated_at__lt=timezone.now() - timedelta(minutes=30)).update(
|
||||
status='FAILED', error='执行中断,请重试', finished_at=timezone.now())
|
||||
next_cleanup = time.monotonic() + 86400
|
||||
while True:
|
||||
close_old_connections()
|
||||
if time.monotonic() >= next_cleanup:
|
||||
from system_manage.ai_hub_assets import cleanup
|
||||
try:
|
||||
cleanup(apply=True)
|
||||
except Exception:
|
||||
self.stderr.write('图片缓存清理失败,将于下次重试')
|
||||
next_cleanup = time.monotonic() + 86400
|
||||
task = claim()
|
||||
if task:
|
||||
execute(task)
|
||||
if options['once']:
|
||||
return
|
||||
if not task:
|
||||
time.sleep(2)
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
# Generated by Django 5.2.8 on 2026-10-19 10:15
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('system_manage', '0006_accesscontrolpolicy_and_record'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='AIFlow',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
|
||||
('owner_id', models.UUIDField(db_index=True)),
|
||||
('name', models.CharField(max_length=128)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('revision', models.PositiveIntegerField(default=1)),
|
||||
('nodes', models.JSONField(default=list)),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AIParserResource',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
|
||||
('owner_id', models.UUIDField(db_index=True)),
|
||||
('name', models.CharField(max_length=128)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('endpoint', models.URLField(max_length=1024)),
|
||||
('credential', models.TextField(blank=True, default='')),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('allowed_users', models.JSONField(default=list)),
|
||||
('options', models.JSONField(default=dict)),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AIService',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
|
||||
('owner_id', models.UUIDField(db_index=True)),
|
||||
('name', models.CharField(max_length=128)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('allowed_users', models.JSONField(default=list)),
|
||||
('key_hash', models.CharField(blank=True, default='', max_length=64)),
|
||||
('requests_per_minute', models.PositiveIntegerField(default=20)),
|
||||
('flow', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='system_manage.aiflow')),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AITask',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
|
||||
('owner_id', models.UUIDField(db_index=True)),
|
||||
('name', models.CharField(max_length=128)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('status', models.CharField(db_index=True, default='PENDING', max_length=16)),
|
||||
('snapshot', models.JSONField(default=dict)),
|
||||
('input_data', models.JSONField(default=dict)),
|
||||
('file_name', models.CharField(default='', max_length=255)),
|
||||
('file_data', models.BinaryField(null=True)),
|
||||
('result', models.JSONField(default=dict)),
|
||||
('steps', models.JSONField(default=list)),
|
||||
('error', models.CharField(default='', max_length=512)),
|
||||
('attempt', models.PositiveIntegerField(default=1)),
|
||||
('started_at', models.DateTimeField(null=True)),
|
||||
('finished_at', models.DateTimeField(null=True)),
|
||||
('service', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aiservice')),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AIArtifact',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128)),
|
||||
('owner_id', models.UUIDField(db_index=True)),
|
||||
('name', models.CharField(max_length=128)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('revision', models.PositiveIntegerField(default=1)),
|
||||
('original', models.TextField(default='')),
|
||||
('markdown', models.TextField(default='')),
|
||||
('layout', models.CharField(default='tree', max_length=32)),
|
||||
('history', models.JSONField(default=list)),
|
||||
('task', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='system_manage.aitask')),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
"""Shared AI resources, versioned flows and durable user results."""
|
||||
import uuid
|
||||
from django.db import models
|
||||
|
||||
|
||||
class HubBase(models.Model):
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
workspace_id = models.CharField(max_length=128, default='default', db_index=True)
|
||||
owner_id = models.UUIDField(db_index=True)
|
||||
name = models.CharField(max_length=128)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class AIParserResource(HubBase):
|
||||
endpoint = models.URLField(max_length=1024)
|
||||
credential = models.TextField(default='', blank=True)
|
||||
enabled = models.BooleanField(default=True)
|
||||
allowed_users = models.JSONField(default=list)
|
||||
options = models.JSONField(default=dict)
|
||||
|
||||
|
||||
class AIFlow(HubBase):
|
||||
revision = models.PositiveIntegerField(default=1)
|
||||
nodes = models.JSONField(default=list)
|
||||
|
||||
|
||||
class AIService(HubBase):
|
||||
flow = models.ForeignKey(AIFlow, on_delete=models.PROTECT)
|
||||
enabled = models.BooleanField(default=True)
|
||||
allowed_users = models.JSONField(default=list)
|
||||
key_hash = models.CharField(max_length=64, default='', blank=True)
|
||||
requests_per_minute = models.PositiveIntegerField(default=20)
|
||||
|
||||
|
||||
class AITask(HubBase):
|
||||
service = models.ForeignKey(AIService, null=True, on_delete=models.SET_NULL)
|
||||
status = models.CharField(max_length=16, default='PENDING', db_index=True)
|
||||
snapshot = models.JSONField(default=dict)
|
||||
input_data = models.JSONField(default=dict)
|
||||
file_name = models.CharField(max_length=255, default='')
|
||||
file_data = models.BinaryField(null=True)
|
||||
result = models.JSONField(default=dict)
|
||||
steps = models.JSONField(default=list)
|
||||
error = models.CharField(max_length=512, default='')
|
||||
attempt = models.PositiveIntegerField(default=1)
|
||||
started_at = models.DateTimeField(null=True)
|
||||
finished_at = models.DateTimeField(null=True)
|
||||
|
||||
|
||||
class AIArtifact(HubBase):
|
||||
task = models.ForeignKey(AITask, null=True, on_delete=models.SET_NULL)
|
||||
revision = models.PositiveIntegerField(default=1)
|
||||
original = models.TextField(default='')
|
||||
markdown = models.TextField(default='')
|
||||
layout = models.CharField(max_length=32, default='tree')
|
||||
history = models.JSONField(default=list)
|
||||
|
|
@ -0,0 +1,462 @@
|
|||
import json
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch, Mock
|
||||
from django.db import transaction
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from rest_framework.exceptions import ValidationError, PermissionDenied, NotFound
|
||||
from users.models import User
|
||||
from users.models.user_manage_ext import UserActivationApplication
|
||||
from system_manage import ai_hub as hub
|
||||
from system_manage.ai_hub_worker import claim, execute
|
||||
from system_manage.models.ai_hub import AIFlow, AIParserResource, AITask, AIArtifact, AIService
|
||||
|
||||
|
||||
class HubTests(TestCase):
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
from django.test import override_settings
|
||||
from pathlib import Path
|
||||
self.asset_temp = tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[3] / '.codex_tmp')
|
||||
self.addCleanup(self.asset_temp.cleanup)
|
||||
self.asset_settings = override_settings(AI_HUB_ASSET_DIR=self.asset_temp.name)
|
||||
self.asset_settings.enable()
|
||||
self.addCleanup(self.asset_settings.disable)
|
||||
self.atomic = transaction.atomic()
|
||||
self.atomic.__enter__()
|
||||
self.admin = User.objects.create(username='admin', nick_name='admin', role='ADMIN', is_active=True)
|
||||
self.user = User.objects.create(username='user', nick_name='user', role='USER', is_active=True)
|
||||
self.other = User.objects.create(username='other', nick_name='other', role='USER', is_active=True)
|
||||
|
||||
def tearDown(self):
|
||||
transaction.set_rollback(True)
|
||||
self.atomic.__exit__(None, None, None)
|
||||
|
||||
def parser(self, **kwargs):
|
||||
return AIParserResource.objects.create(workspace_id='default', owner_id=self.admin.id,
|
||||
name='测试解析', endpoint='http://parser.test/api', allowed_users=[str(self.user.id)], **kwargs)
|
||||
|
||||
def test_parser_grant_and_workspace_isolation(self):
|
||||
p = self.parser()
|
||||
self.assertEqual(hub.one('parsers', p.id, 'default', self.user).id, p.id)
|
||||
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'default', self.other)
|
||||
with self.assertRaises(NotFound): hub.one('parsers', p.id, 'other', self.admin)
|
||||
with self.assertRaises(PermissionDenied): hub.one('parsers', p.id, 'default', self.user, edit=True)
|
||||
|
||||
def test_only_admin_can_register_parser(self):
|
||||
with self.assertRaises(PermissionDenied): hub.parser_save('default', self.user, {})
|
||||
|
||||
def test_authorization_excludes_rejected_pending_disabled(self):
|
||||
application = UserActivationApplication.objects.create(user=self.user, status='REJECTED')
|
||||
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
||||
application.status = 'PENDING'; application.save()
|
||||
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
||||
application.status = 'APPROVED'; application.save()
|
||||
self.assertEqual(hub.users([str(self.user.id)]), [str(self.user.id)])
|
||||
self.user.is_active = False; self.user.save()
|
||||
with self.assertRaises(ValidationError): hub.users([str(self.user.id)])
|
||||
|
||||
def test_parser_secret_never_serialized_and_preserved(self):
|
||||
with patch('common.utils.rsa_util.rsa_long_encrypt', return_value='encrypted'):
|
||||
p = hub.parser_save('default', self.admin, {'name': '解析', 'endpoint': 'http://parser.test/api', 'api_key': 'private', 'allowed_users': []})
|
||||
self.assertEqual(p.credential, 'encrypted')
|
||||
self.assertNotIn('private', json.dumps(hub.serialize(p)))
|
||||
self.assertNotIn('encrypted', json.dumps(hub.serialize(p)))
|
||||
p = hub.parser_save('default', self.admin, {'name': '解析2'}, p)
|
||||
self.assertEqual(p.credential, 'encrypted')
|
||||
|
||||
def test_parser_url_and_options_validation(self):
|
||||
for endpoint in ('file:///tmp/a', 'http://user:secret@server/api', 'http://server/api?key=secret'):
|
||||
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': endpoint})
|
||||
with self.assertRaises(ValidationError): hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/api', 'options': {'server_url': 'http://other'}})
|
||||
|
||||
def test_parser_toggle_preserves_shared_options_and_grants(self):
|
||||
p = hub.parser_save('default', self.admin, {'name': 'shared', 'endpoint': 'http://server/v1',
|
||||
'allowed_users': [str(self.user.id)], 'options': {'protocol': 'v1', 'tier': 'basic'}})
|
||||
self.user.is_active = False
|
||||
self.user.save(update_fields=['is_active'])
|
||||
hub.parser_save('default', self.admin, {'enabled': False}, p)
|
||||
p.refresh_from_db()
|
||||
self.assertFalse(p.enabled)
|
||||
self.assertEqual(p.allowed_users, [str(self.user.id)])
|
||||
self.assertEqual(p.options, {'protocol': 'v1', 'tier': 'basic'})
|
||||
self.assertEqual(p.name, 'shared')
|
||||
|
||||
@patch('requests.get')
|
||||
def test_parser_connection_checks_protocol_and_does_not_return_service_content(self, get):
|
||||
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1', 'options': {'protocol': 'v1'}})
|
||||
get.return_value = Mock(status_code=200)
|
||||
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'post': {}}}, 'secret': 'not-for-browser'}
|
||||
result = hub.parser_test(p)
|
||||
self.assertTrue(result['connected'])
|
||||
self.assertNotIn('secret', result)
|
||||
self.assertEqual(get.call_args.args[0], 'http://server/openapi.json')
|
||||
self.assertFalse(get.call_args.kwargs['allow_redirects'])
|
||||
get.return_value.json.return_value = {'paths': {'/v1/tasks': {'get': {}}}}
|
||||
with self.assertRaises(ValidationError): hub.parser_test(p)
|
||||
p.endpoint = 'http://server/api'; p.options = {'protocol': 'legacy'}
|
||||
get.return_value.json.return_value = {'paths': {'/file_parse': {'post': {}}}}
|
||||
self.assertTrue(hub.parser_test(p)['connected'])
|
||||
|
||||
@patch('requests.get')
|
||||
def test_parser_connection_failures_and_permissions(self, get):
|
||||
import requests
|
||||
from types import SimpleNamespace
|
||||
from system_manage.views.ai_hub import ParserTest
|
||||
p = hub.parser_save('default', self.admin, {'name': 'p', 'endpoint': 'http://server/v1'})
|
||||
with self.assertRaises(PermissionDenied): ParserTest().post(SimpleNamespace(user=self.user), 'default', p.id)
|
||||
get.assert_not_called()
|
||||
get.return_value = Mock(status_code=302)
|
||||
with self.assertRaises(ValidationError): hub.parser_test(p)
|
||||
get.return_value = Mock(status_code=200)
|
||||
get.return_value.json.side_effect = ValueError('sensitive server response')
|
||||
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
|
||||
self.assertNotIn('sensitive', str(error.exception))
|
||||
get.side_effect = requests.Timeout('sensitive URL')
|
||||
with self.assertRaises(ValidationError) as error: hub.parser_test(p)
|
||||
self.assertNotIn('sensitive', str(error.exception))
|
||||
|
||||
def test_flow_versions_are_immutable(self):
|
||||
a = hub.flow_create('default', self.user, {'name': '文档', 'nodes': [{'type': 'render'}]})
|
||||
b = hub.flow_create('default', self.user, {'name': '文档', 'previous_id': str(a.id), 'nodes': [{'type': 'render'}]})
|
||||
self.assertEqual((a.revision, b.revision), (1, 2))
|
||||
self.assertNotEqual(a.id, b.id)
|
||||
|
||||
def test_invalid_flow_and_ungranted_parser_rejected(self):
|
||||
p = self.parser()
|
||||
for nodes in ([{'type': 'render'}, {'type': 'parse', 'resource_id': str(p.id)}], [{'type': 'custom'}], [{'type': 'render'}, {'type': 'render'}]):
|
||||
with self.assertRaises(ValidationError): hub.validate_nodes('default', self.user, nodes)
|
||||
with self.assertRaises(NotFound): hub.validate_nodes('default', self.other, [{'type': 'parse', 'resource_id': str(p.id)}, {'type': 'render'}])
|
||||
|
||||
def test_submit_claim_and_rule_render(self):
|
||||
task = hub.submit('default', self.user, {'markdown': '# 标题\n## 内容\n事实42\n```\n# 不作标题\n```', 'nodes': [{'type': 'render'}]})
|
||||
picked = claim()
|
||||
self.assertEqual(picked.id, task.id)
|
||||
self.assertIsNone(claim())
|
||||
execute(picked); picked.refresh_from_db()
|
||||
self.assertEqual(picked.status, 'SUCCESS')
|
||||
tree = picked.result['tree']
|
||||
self.assertEqual(tree['children'][0]['text'], '标题')
|
||||
self.assertNotIn('不作标题', json.dumps(tree, ensure_ascii=False))
|
||||
|
||||
def test_parser_task_preserves_original_and_source(self):
|
||||
parser = self.parser()
|
||||
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('scan.pdf', b'pdf'))
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {'results': {'scan': {'md_content': '# 内容\n数值42', 'middle_json': {'pages': [1]}}}}
|
||||
with patch('requests.post', return_value=response) as post:
|
||||
execute(claim())
|
||||
self.assertFalse(post.call_args.kwargs['allow_redirects'])
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, 'SUCCESS')
|
||||
self.assertEqual(task.result['original'], '# 内容\n数值42')
|
||||
self.assertEqual(task.result['source']['pages'], [1])
|
||||
|
||||
def test_parser_disabled_or_changed_after_submit_fails(self):
|
||||
parser = self.parser()
|
||||
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
||||
parser.enabled = False; parser.save()
|
||||
with patch('requests.post') as post: execute(claim()); post.assert_not_called()
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, 'FAILED')
|
||||
|
||||
def test_model_calls_existing_provider_with_selected_parameters(self):
|
||||
from models_provider.models import Model
|
||||
model = Model.objects.create(name='整理', model_type='LLM', model_name='demo', provider='test', user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
||||
task = hub.submit('default', self.admin, {'markdown': '# 原稿', 'nodes': [{'type': 'organize', 'model_id': str(model.id), 'temperature': 0}, {'type': 'render'}]})
|
||||
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理稿')
|
||||
with patch('models_provider.tools.get_model', return_value=fake) as get:
|
||||
execute(claim())
|
||||
self.assertEqual(get.call_args.kwargs['temperature'], 0)
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.result['original'], '# 原稿')
|
||||
self.assertEqual(task.result['markdown'], '# 整理稿')
|
||||
|
||||
def test_provider_exception_is_sanitized(self):
|
||||
parser = self.parser()
|
||||
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
||||
with patch('requests.post', side_effect=RuntimeError('private-url-key')): execute(claim())
|
||||
task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
|
||||
self.assertNotIn('private', task.error)
|
||||
|
||||
def test_empty_parse_does_not_succeed(self):
|
||||
parser = self.parser()
|
||||
response = Mock(status_code=200); response.json.return_value = {'results': {'doc': {'md_content': ''}}}
|
||||
with patch('requests.post', return_value=response):
|
||||
with self.assertRaises(ValidationError): hub.parse_file(parser, 'x.pdf', b'pdf')
|
||||
|
||||
def test_task_validation_error_is_displayed_without_framework_details(self):
|
||||
parser = self.parser()
|
||||
task = hub.submit('default', self.user, {'nodes': [{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('x.pdf', b'pdf'))
|
||||
with patch('system_manage.ai_hub.parse_file', side_effect=ValidationError('解析失败,请检查文档')):
|
||||
execute(claim())
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.error, '解析失败,请检查文档')
|
||||
|
||||
def test_artifact_review_preserves_original_and_detects_conflicts(self):
|
||||
a = hub.artifact_save('default', self.user, {'name': '图谱', 'original': '# OCR原值', 'markdown': '# 复核结果'})
|
||||
a = hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'original': '篡改原稿', 'markdown': '# 第二版'}, a)
|
||||
self.assertEqual(a.original, '# OCR原值'); self.assertEqual(a.revision, 2)
|
||||
self.assertEqual(a.history[0]['markdown'], '# 复核结果')
|
||||
with self.assertRaises(ValidationError): hub.artifact_save('default', self.user, {'name': '图谱', 'revision': 1, 'markdown': '# 旧提交'}, a)
|
||||
with self.assertRaises(NotFound): hub.one('artifacts', a.id, 'default', self.other)
|
||||
|
||||
def test_published_service_is_version_pinned_and_disabled_stops_submit(self):
|
||||
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
||||
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
||||
task = hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
||||
self.assertEqual(task.snapshot['flow_revision'], 1)
|
||||
service.enabled = False; service.save()
|
||||
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
||||
execute(claim()); task.refresh_from_db(); self.assertEqual(task.status, 'FAILED')
|
||||
|
||||
def test_unknown_ids_and_missing_inputs_are_validation_errors(self):
|
||||
with self.assertRaises(ValidationError): hub.one('parsers', None, 'default', self.admin)
|
||||
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'nodes': [{'type': 'render'}]})
|
||||
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': 'x', 'nodes': [{'type': 'parse', 'resource_id': str(self.parser().id)}, {'type': 'render'}]})
|
||||
|
||||
def api(self, method, kind, data=None, pk=None, user=None, headers=None):
|
||||
from rest_framework.test import APIRequestFactory, force_authenticate
|
||||
from system_manage.views.ai_hub import HubView, PublishedRun
|
||||
request = getattr(APIRequestFactory(), method)('/', data or {}, format='json', **(headers or {}))
|
||||
force_authenticate(request, user=user or self.admin)
|
||||
kwargs = {'workspace_id': 'default'}
|
||||
if kind != 'published': kwargs['kind'] = kind
|
||||
if pk: kwargs['pk'] = pk
|
||||
view = PublishedRun if kind == 'published' else HubView
|
||||
with patch('users.serializers.user.get_workspace_list_by_user', return_value=[{'id': 'default'}]):
|
||||
response = view.as_view()(request, **kwargs)
|
||||
return json.loads(response.content) if hasattr(response, 'content') else response.data
|
||||
|
||||
def test_api_service_key_rotation_revocation_and_real_task(self):
|
||||
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
||||
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
||||
rotated = self.api('post', 'services', {'rotate_key': True}, service.id)['data']
|
||||
self.assertTrue(rotated['api_key'])
|
||||
self.assertNotIn('api_key', self.api('get', 'services', pk=service.id)['data'])
|
||||
denied = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
|
||||
self.assertNotEqual(denied['code'], 200)
|
||||
accepted = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user, {'HTTP_X_SERVICE_KEY': rotated['api_key']})
|
||||
self.assertEqual(accepted['code'], 200)
|
||||
self.assertEqual(AITask.objects.get(id=accepted['data']['id']).status, 'PENDING')
|
||||
self.api('post', 'services', {'revoke_key': True}, service.id)
|
||||
service.refresh_from_db(); self.assertEqual(service.key_hash, '')
|
||||
|
||||
def test_api_cannot_edit_shared_service_and_stops_disabled_service(self):
|
||||
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
||||
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
||||
response = self.api('post', 'services', {'enabled': False}, service.id, self.user)
|
||||
self.assertNotEqual(response['code'], 200)
|
||||
self.api('post', 'services', {'enabled': False}, service.id)
|
||||
response = self.api('post', 'published', {'markdown': '# 文档'}, service.id, self.user)
|
||||
self.assertNotEqual(response['code'], 200)
|
||||
|
||||
def test_api_retry_resets_failure_without_duplicate_task(self):
|
||||
task = hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
|
||||
task.status = 'FAILED'; task.error = '中断'; task.save()
|
||||
response = self.api('post', 'tasks', {}, task.id, self.user)
|
||||
self.assertEqual(response['code'], 200)
|
||||
task.refresh_from_db(); self.assertEqual(task.attempt, 2); self.assertEqual(task.status, 'PENDING')
|
||||
self.assertEqual(AITask.objects.count(), 1)
|
||||
response = self.api('post', 'tasks', {}, task.id, self.user)
|
||||
self.assertNotEqual(response['code'], 200)
|
||||
|
||||
def test_knowledge_binding_and_shared_parser_execution(self):
|
||||
from knowledge.models import KnowledgeFolder, Knowledge
|
||||
from system_manage.ai_hub_knowledge import parser_binding, split_with_parser
|
||||
KnowledgeFolder.objects.create(id='default', name='root')
|
||||
knowledge = Knowledge.objects.create(name='测试知识库', desc='', folder_id='default', meta={'segmentation': {'chunk_size': 100}})
|
||||
parser = self.parser()
|
||||
parser_binding('default', knowledge.id, self.user, {'parser_id': str(parser.id)})
|
||||
knowledge.refresh_from_db(); self.assertEqual(knowledge.meta['segmentation']['chunk_size'], 100)
|
||||
with self.assertRaises(NotFound): parser_binding('default', knowledge.id, self.other, {'parser_id': str(parser.id)})
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {'results': {'scan': {'md_content': '# 章节\n识别结果42', 'middle_json': {'pages': [1]}}}}
|
||||
file = SimpleUploadedFile('scan.pdf', b'pdf')
|
||||
with patch('requests.post', return_value=response):
|
||||
result, provenance = split_with_parser('default', knowledge.id, self.user, file, None, False, 100, 20)
|
||||
task = AITask.objects.get(id=provenance['task_id'])
|
||||
self.assertEqual(task.status, 'SUCCESS'); self.assertEqual(provenance['resource_id'], str(parser.id))
|
||||
self.assertIn('识别结果42', json.dumps(result, ensure_ascii=False)); self.assertEqual(file.tell(), 0)
|
||||
parser_binding('default', knowledge.id, self.user, {'parser_id': ''})
|
||||
self.assertIsNone(split_with_parser('default', knowledge.id, None, file, None, False, 100, 0))
|
||||
|
||||
def test_model_selection_follows_view_grants_instead_of_ownership(self):
|
||||
from models_provider.models import Model
|
||||
from types import SimpleNamespace
|
||||
model = Model.objects.create(name='私有', model_type='LLM', model_name='demo', provider='test', user_id=self.user.id, credential='', status='SUCCESS', model_params_form=[])
|
||||
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[]):
|
||||
with self.assertRaises(PermissionDenied): hub.model_for('default', self.user, model.id)
|
||||
grant = SimpleNamespace(target=str(model.id), permission_list=['VIEW'])
|
||||
with patch('system_manage.models.WorkspaceUserResourcePermission.objects.filter', return_value=[grant]):
|
||||
self.assertEqual(hub.model_for('default', self.user, model.id).id, model.id)
|
||||
|
||||
def test_queue_limit_applies_to_published_service_too(self):
|
||||
flow = hub.flow_create('default', self.admin, {'name': '流程', 'nodes': [{'type': 'render'}]})
|
||||
service = hub.service_create('default', self.admin, {'name': '服务', 'flow_id': str(flow.id), 'allowed_users': [str(self.user.id)]})
|
||||
for _ in range(10):
|
||||
hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
||||
with self.assertRaises(ValidationError): hub.submit('default', self.user, {'markdown': '# 文档'}, service=service)
|
||||
self.assertEqual(AITask.objects.count(), 10)
|
||||
|
||||
def test_stopped_user_cannot_submit_directly(self):
|
||||
self.user.is_active = False; self.user.save()
|
||||
with self.assertRaises(PermissionDenied): hub.submit('default', self.user, {'markdown': '# 文档', 'nodes': [{'type': 'render'}]})
|
||||
|
||||
def test_v1_adapter_polls_registered_endpoint_and_preserves_source(self):
|
||||
parser = self.parser(options={'protocol': 'v1', 'tier': 'flash'})
|
||||
created = Mock(status_code=202); created.json.return_value = {'task_id': 'task_test', 'status': 'queued', 'result_url': 'http://untrusted/secret'}
|
||||
status = Mock(status_code=200); status.json.return_value = {'task_id': 'task_test', 'status': 'completed'}
|
||||
result = Mock(status_code=200); result.json.return_value = {'files': [{'status': 'completed', 'parse': {'engine': 'native'}, 'content': {'markdown': '# 识别结果42', 'middle_json': {'pages': [1]}}}]}
|
||||
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=created) as post, patch('system_manage.ai_hub_parser_v1.requests.get', side_effect=[status, result]) as get, patch('system_manage.ai_hub_parser_v1.time.sleep'):
|
||||
parsed = hub.parse_file(parser, 'test.pdf', b'pdf')
|
||||
self.assertEqual(parsed['markdown'], '# 识别结果42')
|
||||
self.assertEqual(parsed['source']['middle_json']['pages'], [1])
|
||||
self.assertEqual(get.call_args.args[0], parser.endpoint + '/tasks/task_test/result')
|
||||
self.assertIn(('tier', 'flash'), post.call_args.kwargs['params'])
|
||||
|
||||
def test_v1_adapter_rejects_external_task_paths_and_failed_files(self):
|
||||
parser = self.parser(options={'protocol': 'v1'})
|
||||
response = Mock(status_code=202); response.json.return_value = {'task_id': '../../other', 'status': 'completed'}
|
||||
with patch('system_manage.ai_hub_parser_v1.requests.post', return_value=response), patch('system_manage.ai_hub_parser_v1.requests.get') as get:
|
||||
with self.assertRaises(ValidationError): hub.parse_file(parser, 'test.pdf', b'pdf')
|
||||
get.assert_not_called()
|
||||
|
||||
def image_markdown(self):
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image
|
||||
content = io.BytesIO()
|
||||
Image.new('RGB', (40, 20), '#2468ac').save(content, 'PNG')
|
||||
return '# 图片验收\n).decode() + ')', content.getvalue()
|
||||
|
||||
def test_asset_submission_persistence_deduplication_and_isolation(self):
|
||||
from system_manage.ai_hub_assets import ASSET, image_bytes, root
|
||||
markdown, content = self.image_markdown()
|
||||
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
||||
url = ASSET.search(task.input_data['markdown'])
|
||||
self.assertIsNotNone(url)
|
||||
self.assertNotIn('base64', task.input_data['markdown'])
|
||||
self.assertEqual(image_bytes('default', self.user, url[2])[0], content)
|
||||
with self.assertRaises(NotFound): image_bytes('default', self.other, url[2])
|
||||
with self.assertRaises(NotFound): image_bytes('other', self.admin, url[2])
|
||||
with self.assertRaises(NotFound): hub.submit('default', self.other, {'markdown': task.input_data['markdown'], 'nodes': [{'type': 'render'}]})
|
||||
hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
||||
self.assertEqual(len(list(root().glob('*.bin'))), 1)
|
||||
|
||||
def test_asset_cleanup_preserves_original_history_and_inflight(self):
|
||||
import time
|
||||
from system_manage.ai_hub_assets import ASSET, cleanup, root
|
||||
markdown, _ = self.image_markdown()
|
||||
artifact = hub.artifact_save('default', self.user, {'name': '测试图片', 'markdown': markdown, 'original': '', 'layout': 'tree'})
|
||||
asset_id = ASSET.search(artifact.markdown)[2]
|
||||
hub.artifact_save('default', self.user, {'name': artifact.name, 'revision': artifact.revision, 'markdown': '# 删除当前引用'}, artifact)
|
||||
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
|
||||
self.assertTrue((root() / (asset_id + '.bin')).exists())
|
||||
# Only remove the isolated test record; existing shared records are never involved.
|
||||
artifact.delete()
|
||||
task = hub.submit('default', self.user, {'markdown': markdown, 'nodes': [{'type': 'render'}]})
|
||||
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 0)
|
||||
execute(task)
|
||||
self.assertEqual(cleanup(now=time.time() + 8 * 86400), 1)
|
||||
self.assertTrue((root() / (asset_id + '.bin')).exists())
|
||||
self.assertEqual(cleanup(apply=True, now=time.time() + 8 * 86400), 1)
|
||||
self.assertFalse((root() / (asset_id + '.bin')).exists())
|
||||
|
||||
def test_asset_parser_relative_images_and_invalid_payload(self):
|
||||
from system_manage.ai_hub_assets import asset_lock, normalize, ASSET
|
||||
markdown, _ = self.image_markdown()
|
||||
uri = markdown.split('](', 1)[1][:-1]
|
||||
with asset_lock():
|
||||
normalized = normalize('\n<img src="a.png">', 'default', self.user, {'a.png': uri})
|
||||
self.assertEqual(len(ASSET.findall(normalized)), 2)
|
||||
with self.assertRaises(ValidationError): normalize('', 'default', self.user)
|
||||
|
||||
def test_asset_bundle_is_offline_and_model_cannot_drop_images(self):
|
||||
import io
|
||||
import zipfile
|
||||
from types import SimpleNamespace
|
||||
from system_manage.views.ai_hub import MarkdownExport
|
||||
from system_manage.ai_hub_assets import protect_images, restore_images
|
||||
markdown, content = self.image_markdown()
|
||||
artifact = hub.artifact_save('default', self.user, {'name': '测试导出', 'markdown': markdown})
|
||||
response = MarkdownExport().post(SimpleNamespace(user=self.user, data={'markdown': artifact.markdown}), 'default')
|
||||
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
||||
self.assertEqual(len(archive.namelist()), 2)
|
||||
exported = archive.read('document.md').decode()
|
||||
self.assertNotIn('/admin/api/', exported)
|
||||
self.assertIn('images/', exported)
|
||||
self.assertEqual(archive.read(next(n for n in archive.namelist() if n.startswith('images/'))), content)
|
||||
protected, images = protect_images(artifact.markdown)
|
||||
self.assertNotIn('/admin/api/', protected)
|
||||
self.assertEqual(restore_images(protected, images), artifact.markdown)
|
||||
self.assertIn(images[0], restore_images('# 整理完成但遗漏图片', images))
|
||||
self.assertEqual(restore_images(protected + protected, images).count(images[0]), 1)
|
||||
self.assertIn(images[0], restore_images('AIHUBIMAGE 0001 TOKEN', images))
|
||||
|
||||
def test_organize_restores_eight_omitted_images_and_preserves_original(self):
|
||||
from models_provider.models import Model
|
||||
markdown, _ = self.image_markdown()
|
||||
image = markdown.split('\n', 1)[1]
|
||||
original = '# 报告\n' + '\n\n'.join(image for _ in range(8))
|
||||
model = Model.objects.create(name='测试整理', model_type='LLM', model_name='demo', provider='test',
|
||||
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
||||
task = hub.submit('default', self.admin, {'markdown': original, 'nodes': [
|
||||
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
|
||||
normalized = task.input_data['markdown']
|
||||
fake = Mock(); fake.invoke.return_value = Mock(content='# 整理结果\n- 结论42')
|
||||
with patch('models_provider.tools.get_model', return_value=fake):
|
||||
execute(claim())
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, 'SUCCESS', task.error)
|
||||
self.assertEqual(task.result['original'], normalized)
|
||||
self.assertEqual(task.result['markdown'].count(', 8)
|
||||
self.assertNotIn('AIHUBIMAGE', task.result['markdown'])
|
||||
self.assertIn('结论42', task.result['markdown'])
|
||||
self.assertNotIn('/admin/api/', fake.invoke.call_args.args[0][1].content)
|
||||
|
||||
def test_failed_organize_keeps_source_available(self):
|
||||
from models_provider.models import Model
|
||||
model = Model.objects.create(name='测试失败', model_type='LLM', model_name='demo', provider='test',
|
||||
user_id=self.admin.id, credential='encrypted', status='SUCCESS', model_params_form=[])
|
||||
task = hub.submit('default', self.admin, {'markdown': '# 保留原稿42', 'nodes': [
|
||||
{'type': 'organize', 'model_id': str(model.id)}, {'type': 'render'}]})
|
||||
with patch('models_provider.tools.get_model', side_effect=RuntimeError('private')): execute(claim())
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(task.status, 'FAILED')
|
||||
self.assertEqual(task.result['original'], '# 保留原稿42')
|
||||
self.assertNotIn('private', task.error)
|
||||
|
||||
def test_single_file_progress_is_indeterminate_and_counts_are_real(self):
|
||||
from system_manage.ai_hub_progress import parser_progress, task_progress
|
||||
single = parser_progress({'status': 'running', 'progress': {'completed': 0, 'total': 1}})
|
||||
self.assertIsNone(single['percent'])
|
||||
self.assertEqual(single['stage'], '正在解析文档')
|
||||
self.assertEqual(parser_progress({'status': 'queued'})['stage'], '等待解析服务处理')
|
||||
self.assertEqual(parser_progress({'status': 'running', 'progress': {'completed': 2, 'total': 4}})['percent'], 50)
|
||||
task = hub.submit('default', self.user, {'markdown': '# 测试', 'nodes': [{'type': 'render'}]})
|
||||
self.assertIsNone(task_progress(task)['percent'])
|
||||
task.status = 'RUNNING'; task.steps = [{'type': 'parse', 'status': 'RUNNING', 'progress': single}]
|
||||
self.assertEqual(hub.serialize(task, True)['progress']['stage'], '正在解析文档')
|
||||
self.assertIsNone(hub.serialize(task, True)['progress']['percent'])
|
||||
|
||||
def test_v1_progress_reaches_database_before_task_finishes(self):
|
||||
parser = self.parser(options={'protocol': 'v1'})
|
||||
task = hub.submit('default', self.user, {'nodes': [
|
||||
{'type': 'parse', 'resource_id': str(parser.id)}, {'type': 'render'}]}, SimpleUploadedFile('test.pdf', b'pdf'))
|
||||
created = Mock(status_code=202); created.json.return_value = {'task_id': 'test_progress', 'status': 'queued'}
|
||||
running = Mock(status_code=200); running.json.return_value = {'status': 'running', 'progress': {'completed': 0, 'total': 1}}
|
||||
done = Mock(status_code=200); done.json.return_value = {'status': 'completed'}
|
||||
result = Mock(status_code=200); result.json.return_value = {'files': [
|
||||
{'status': 'completed', 'content': {'markdown': '# 完成42'}}]}
|
||||
seen = []
|
||||
responses = iter([running, done, result])
|
||||
def get(*args, **kwargs):
|
||||
row = AITask.objects.get(pk=task.pk)
|
||||
seen.append(hub.serialize(row, True)['progress'])
|
||||
return next(responses)
|
||||
with patch('requests.post', return_value=created), patch('requests.get', side_effect=get), patch('system_manage.ai_hub_parser_v1.time.sleep'):
|
||||
execute(claim())
|
||||
self.assertEqual([item['stage'] for item in seen], ['等待解析服务处理', '正在解析文档', '获取解析结果'])
|
||||
self.assertTrue(all(item['percent'] is None for item in seen))
|
||||
task.refresh_from_db()
|
||||
self.assertEqual(hub.serialize(task, True)['progress']['percent'], 100)
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
from .views.ai_hub import HubView, PublishedRun, ParserTest
|
||||
from .views.ai_hub import HubView, PublishedRun, ParserTest, AssetContent, MarkdownExport
|
||||
|
||||
app_name = "system_manage"
|
||||
# @formatter:off
|
||||
urlpatterns = [
|
||||
path('workspace/<str:workspace_id>/ai_hub/exports', MarkdownExport.as_view()),
|
||||
path('workspace/<str:workspace_id>/ai_hub/assets/<uuid:pk>/content', AssetContent.as_view()),
|
||||
path('workspace/<str:workspace_id>/ai_hub/parsers/<uuid:pk>/test', ParserTest.as_view()),
|
||||
path('workspace/<str:workspace_id>/ai_hub/published/<uuid:pk>/run', PublishedRun.as_view()),
|
||||
path('workspace/<str:workspace_id>/ai_hub/<str:kind>', HubView.as_view()),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,201 @@
|
|||
import hashlib
|
||||
import secrets
|
||||
from django.core.cache import cache
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from rest_framework.views import APIView
|
||||
from rest_framework.exceptions import ValidationError, PermissionDenied
|
||||
from common.auth import TokenAuth
|
||||
from common.result import result
|
||||
from system_manage import ai_hub as hub
|
||||
from system_manage.models.ai_hub import AIService, AITask
|
||||
|
||||
|
||||
class HubView(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
def initial(self, request, *args, **kwargs):
|
||||
super().initial(request, *args, **kwargs)
|
||||
from users.serializers.user import get_workspace_list_by_user
|
||||
if not request.user.is_active or kwargs['workspace_id'] not in {str(w['id']) for w in get_workspace_list_by_user(request.user.id)}:
|
||||
raise PermissionDenied('无权访问')
|
||||
|
||||
def get(self, request, workspace_id, kind, pk=None):
|
||||
if kind == 'resources':
|
||||
parsers = hub.visible(hub.AIParserResource.objects.filter(workspace_id=workspace_id, enabled=True), request.user)
|
||||
from users.models import User
|
||||
from users.models.user_manage_ext import UserActivationApplication
|
||||
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
|
||||
return result.success({'parsers': [hub.serialize(p) for p in parsers],
|
||||
'models': list(hub.authorized_models(workspace_id, request.user).values('id', 'name', 'model_type')),
|
||||
'users': [{'id': str(u.id), 'name': u.nick_name or u.username} for u in User.objects.filter(is_active=True).exclude(id__in=pending)] if hub.admin(request.user) else [],
|
||||
'is_admin': hub.admin(request.user)})
|
||||
if pk:
|
||||
obj = hub.one(kind, pk, workspace_id, request.user)
|
||||
data = hub.serialize(obj, detail=True)
|
||||
if kind == 'parsers' and hub.admin(request.user):
|
||||
data['endpoint'] = obj.endpoint
|
||||
return result.success(data)
|
||||
if kind not in hub.KINDS:
|
||||
raise ValidationError('资源类型无效')
|
||||
query = hub.visible(hub.KINDS[kind].objects.filter(workspace_id=workspace_id), request.user)
|
||||
if kind == 'tasks':
|
||||
query = query.defer('file_data', 'input_data', 'snapshot', 'result')
|
||||
elif kind == 'artifacts':
|
||||
query = query.defer('original', 'markdown', 'history')
|
||||
return result.success([hub.serialize(obj) for obj in query.order_by('-created_at')[:200]])
|
||||
|
||||
def post(self, request, workspace_id, kind, pk=None):
|
||||
data = request.data
|
||||
if request.FILES:
|
||||
import json
|
||||
try:
|
||||
data = json.loads(request.data.get('data', '{}'))
|
||||
except (ValueError, TypeError):
|
||||
raise ValidationError('任务参数无效')
|
||||
if not isinstance(data, dict):
|
||||
raise ValidationError('请求参数无效')
|
||||
if pk:
|
||||
with transaction.atomic():
|
||||
obj = hub.one(kind, pk, workspace_id, request.user, edit=True)
|
||||
if kind == 'parsers':
|
||||
obj = hub.parser_save(workspace_id, request.user, data, obj)
|
||||
elif kind == 'artifacts':
|
||||
obj = hub.artifact_save(workspace_id, request.user, data, obj)
|
||||
elif kind == 'services':
|
||||
hub.require_admin(request.user)
|
||||
obj = AIService.objects.select_for_update().get(pk=obj.pk)
|
||||
if 'name' in data:
|
||||
obj.name = hub.text(data['name'], '名称', limit=128)
|
||||
if 'requests_per_minute' in data:
|
||||
obj.requests_per_minute = hub.integer(data['requests_per_minute'], 1, 600, '调用频率')
|
||||
if 'enabled' in data:
|
||||
if not isinstance(data['enabled'], bool):
|
||||
raise ValidationError('启用状态无效')
|
||||
obj.enabled = data['enabled']
|
||||
if 'allowed_users' in data:
|
||||
obj.allowed_users = hub.users(data['allowed_users'])
|
||||
key = secrets.token_urlsafe(32) if data.get('rotate_key') else None
|
||||
if key:
|
||||
obj.key_hash = hashlib.sha256(key.encode()).hexdigest()
|
||||
if data.get('revoke_key'):
|
||||
obj.key_hash = ''
|
||||
obj.save()
|
||||
output = hub.serialize(obj)
|
||||
if key:
|
||||
output['api_key'] = key
|
||||
return result.success(output)
|
||||
elif kind == 'tasks':
|
||||
obj = AITask.objects.select_for_update().get(pk=obj.pk)
|
||||
if obj.status != 'FAILED':
|
||||
raise ValidationError('仅失败任务可重试')
|
||||
if hub.make_snapshot(workspace_id, request.user, obj.snapshot['nodes']) != obj.snapshot['nodes']:
|
||||
raise ValidationError('资源版本已变更,请重新提交任务')
|
||||
if obj.service_id and not obj.service.enabled:
|
||||
raise ValidationError('服务已停用')
|
||||
obj.status, obj.error, obj.steps = 'PENDING', '', []
|
||||
obj.attempt += 1
|
||||
obj.started_at = obj.finished_at = None
|
||||
obj.save()
|
||||
else:
|
||||
raise ValidationError('流程版本不可修改,请另存版本')
|
||||
return result.success(hub.serialize(obj, detail=True))
|
||||
if kind == 'parsers':
|
||||
obj = hub.parser_save(workspace_id, request.user, data)
|
||||
elif kind == 'flows':
|
||||
obj = hub.flow_create(workspace_id, request.user, data)
|
||||
elif kind == 'services':
|
||||
obj = hub.service_create(workspace_id, request.user, data)
|
||||
elif kind == 'tasks':
|
||||
# Bound pending backlog per user; GPU work is consumed by the isolated worker.
|
||||
if AITask.objects.filter(owner_id=request.user.id, status__in=['PENDING', 'RUNNING']).count() >= 10:
|
||||
raise ValidationError('待处理任务已达上限')
|
||||
obj = hub.submit(workspace_id, request.user, data, request.FILES.get('file'))
|
||||
elif kind == 'artifacts':
|
||||
obj = hub.artifact_save(workspace_id, request.user, data)
|
||||
else:
|
||||
raise ValidationError('资源类型无效')
|
||||
return result.success(hub.serialize(obj, detail=True))
|
||||
|
||||
|
||||
class ParserTest(HubView):
|
||||
http_method_names = ['post', 'options']
|
||||
|
||||
def post(self, request, workspace_id, pk):
|
||||
hub.require_admin(request.user)
|
||||
parser = hub.one('parsers', pk, workspace_id, request.user)
|
||||
return result.success(hub.parser_test(parser))
|
||||
|
||||
|
||||
class AssetContent(HubView):
|
||||
http_method_names = ['get', 'options']
|
||||
|
||||
def get(self, request, workspace_id, pk):
|
||||
from django.http import HttpResponse
|
||||
from system_manage.ai_hub_assets import image_bytes
|
||||
content, meta = image_bytes(workspace_id, request.user, pk)
|
||||
response = HttpResponse(content, content_type=meta['mime'])
|
||||
response['Cache-Control'] = 'private, no-store'
|
||||
response['X-Content-Type-Options'] = 'nosniff'
|
||||
return response
|
||||
|
||||
|
||||
class MarkdownExport(HubView):
|
||||
http_method_names = ['post', 'options']
|
||||
|
||||
def post(self, request, workspace_id):
|
||||
import io
|
||||
import zipfile
|
||||
from django.http import HttpResponse
|
||||
from system_manage.ai_hub_assets import ASSET, asset_lock, image_bytes, normalize
|
||||
markdown = hub.text(request.data.get('markdown'), '文档内容')
|
||||
with asset_lock():
|
||||
markdown = normalize(markdown, workspace_id, request.user)
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive:
|
||||
replacements = {}
|
||||
total = 0
|
||||
for match in ASSET.finditer(markdown):
|
||||
if match[0] in replacements:
|
||||
continue
|
||||
content, meta = image_bytes(workspace_id, request.user, match[2])
|
||||
total += len(content)
|
||||
if total > 100 * 1024 * 1024:
|
||||
raise ValidationError('导出图片总大小超过限制')
|
||||
extension = meta['mime'].split('/')[1]
|
||||
name = f'images/{match[2]}.{extension}'
|
||||
archive.writestr(name, content)
|
||||
replacements[match[0]] = name
|
||||
for url, name in replacements.items():
|
||||
markdown = markdown.replace(url, name)
|
||||
archive.writestr('document.md', markdown)
|
||||
response = HttpResponse(buffer.getvalue(), content_type='application/zip')
|
||||
response['Content-Disposition'] = 'attachment; filename="markdown.zip"'
|
||||
return response
|
||||
|
||||
|
||||
class PublishedRun(HubView):
|
||||
"""Internal service: login AND explicit resource grants; Key is an optional extra credential."""
|
||||
authentication_classes = [TokenAuth]
|
||||
http_method_names = ['post', 'options']
|
||||
|
||||
def post(self, request, workspace_id, pk):
|
||||
service = hub.one('services', pk, workspace_id, request.user)
|
||||
if service.key_hash:
|
||||
supplied = hashlib.sha256(request.headers.get('X-Service-Key', '').encode()).hexdigest()
|
||||
if not secrets.compare_digest(supplied, service.key_hash):
|
||||
raise PermissionDenied('服务Key无效')
|
||||
minute = int(timezone.now().timestamp()) // 60
|
||||
key = f'ai-hub-rate:{service.id}:{minute}'
|
||||
cache.add(key, 0, 70)
|
||||
if cache.incr(key) > service.requests_per_minute:
|
||||
raise ValidationError('调用过于频繁')
|
||||
data = request.data
|
||||
if request.FILES:
|
||||
import json
|
||||
try:
|
||||
data = json.loads(request.data.get('data', '{}'))
|
||||
except (ValueError, TypeError):
|
||||
raise ValidationError('任务参数无效')
|
||||
task = hub.submit(workspace_id, request.user, data, request.FILES.get('file'), service=service)
|
||||
return result.success(hub.serialize(task))
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# Generated for user activation audit and role assignment
|
||||
import uuid_utils.compat
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('users', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='UserActivationApplication',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
|
||||
('email', models.EmailField(blank=True, default='', max_length=254, verbose_name='注册邮箱')),
|
||||
('email_verified', models.BooleanField(default=False, verbose_name='邮箱是否已验证')),
|
||||
('status', models.CharField(choices=[('PENDING', '待审核'), ('APPROVED', '已通过'), ('REJECTED', '已拒绝')], db_index=True, default='PENDING', max_length=20, verbose_name='审核状态')),
|
||||
('applied_at', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='申请时间')),
|
||||
('audit_time', models.DateTimeField(blank=True, null=True, verbose_name='审核时间')),
|
||||
('reject_reason', models.CharField(blank=True, default='', max_length=500, verbose_name='拒绝原因')),
|
||||
('audit_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='审核人')),
|
||||
('user', models.OneToOneField(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='activation_application', to='users.user', verbose_name='关联用户')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'user_activation_application',
|
||||
'ordering': ['-applied_at'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='UserRoleAssignment',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
|
||||
('role_id', models.CharField(db_index=True, default='USER', max_length=64, verbose_name='角色标识')),
|
||||
('workspace_id', models.CharField(db_index=True, default='default', max_length=128, verbose_name='工作空间id')),
|
||||
('granted_at', models.DateTimeField(auto_now_add=True, verbose_name='授权时间')),
|
||||
('granted_by', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='授权人')),
|
||||
('user', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='role_assignments', to='users.user', verbose_name='用户')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'user_role_assignment',
|
||||
'ordering': ['granted_at'],
|
||||
'unique_together': {('user', 'role_id', 'workspace_id')},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# coding=utf-8
|
||||
"""
|
||||
用户管理扩展模型:
|
||||
1. UserActivationApplication:初次注册用户的待激活审核申请(独立数据对象)
|
||||
2. UserRoleAssignment:用户-角色-工作空间授权关系(独立数据对象)
|
||||
|
||||
说明:这两个模型用于规格书 2.2.2「设置初次注册用户为激活用户」「更改用户角色」
|
||||
两个功能过程的独立申报,避免与「启用/停用用户账号」「更改用户信息」共用同一数据对象。
|
||||
"""
|
||||
import uuid_utils.compat as uuid
|
||||
|
||||
from django.db import models
|
||||
|
||||
|
||||
class ActivationStatus(models.TextChoices):
|
||||
PENDING = "PENDING", "待审核"
|
||||
APPROVED = "APPROVED", "已通过"
|
||||
REJECTED = "REJECTED", "已拒绝"
|
||||
|
||||
|
||||
class UserActivationApplication(models.Model):
|
||||
"""初次注册用户的激活审核申请。"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
|
||||
user = models.OneToOneField('users.User', on_delete=models.CASCADE, db_constraint=False,
|
||||
related_name="activation_application", verbose_name="关联用户")
|
||||
email = models.EmailField(verbose_name="注册邮箱", blank=True, default="")
|
||||
email_verified = models.BooleanField(default=False, verbose_name="邮箱是否已验证")
|
||||
status = models.CharField(max_length=20, choices=ActivationStatus.choices,
|
||||
default=ActivationStatus.PENDING, db_index=True, verbose_name="审核状态")
|
||||
applied_at = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="申请时间")
|
||||
audit_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False,
|
||||
null=True, blank=True, related_name="+", verbose_name="审核人")
|
||||
audit_time = models.DateTimeField(null=True, blank=True, verbose_name="审核时间")
|
||||
reject_reason = models.CharField(max_length=500, blank=True, default="", verbose_name="拒绝原因")
|
||||
|
||||
class Meta:
|
||||
db_table = "user_activation_application"
|
||||
ordering = ["-applied_at"]
|
||||
|
||||
|
||||
class UserRoleAssignment(models.Model):
|
||||
"""用户-角色-工作空间授权关系。"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
|
||||
user = models.ForeignKey('users.User', on_delete=models.CASCADE, db_constraint=False,
|
||||
related_name="role_assignments", verbose_name="用户")
|
||||
role_id = models.CharField(max_length=64, default="USER", db_index=True, verbose_name="角色标识")
|
||||
workspace_id = models.CharField(max_length=128, default="default", db_index=True, verbose_name="工作空间id")
|
||||
granted_by = models.ForeignKey('users.User', on_delete=models.SET_NULL, db_constraint=False,
|
||||
null=True, blank=True, related_name="+", verbose_name="授权人")
|
||||
granted_at = models.DateTimeField(auto_now_add=True, verbose_name="授权时间")
|
||||
|
||||
class Meta:
|
||||
db_table = "user_role_assignment"
|
||||
unique_together = ("user", "role_id", "workspace_id")
|
||||
ordering = ["granted_at"]
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import json
|
||||
|
||||
from rest_framework import serializers
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from common.exception.app_exception import AppApiException
|
||||
from common.utils.common import password_encrypt
|
||||
from common.utils.rsa_util import decrypt
|
||||
from users.models import User, UserActivationApplication, ActivationStatus
|
||||
from users.serializers.user import UserManageSerializer
|
||||
|
||||
|
||||
class RegisterSerializer(UserManageSerializer.UserInstance):
|
||||
re_password = serializers.CharField(required=True, write_only=True, trim_whitespace=False)
|
||||
|
||||
def get_fields(self):
|
||||
fields = super().get_fields()
|
||||
for name in ('source', 'pending_activation'):
|
||||
fields.pop(name)
|
||||
return fields
|
||||
|
||||
def is_valid(self, *, raise_exception=True):
|
||||
# 重复账号的处理由 register 在锁定申请后按审核状态判断。
|
||||
return serializers.Serializer.is_valid(self, raise_exception=raise_exception)
|
||||
|
||||
def validate(self, attrs):
|
||||
if attrs['password'] != attrs['re_password']:
|
||||
raise serializers.ValidationError({'re_password': '两次密码不一致'})
|
||||
return attrs
|
||||
|
||||
@staticmethod
|
||||
def _match_condition(username: str, email: str) -> Q:
|
||||
condition = Q(username=username)
|
||||
if email:
|
||||
condition = condition | Q(email=email)
|
||||
return condition
|
||||
|
||||
@staticmethod
|
||||
def _re_register(existing_user: User, data: dict) -> dict:
|
||||
"""被拒绝的用户修改注册信息后重新提交:清空拒绝原因,回到待审核。"""
|
||||
condition = RegisterSerializer._match_condition(data['username'], data.get('email', ''))
|
||||
condition |= Q(nick_name=data['nick_name'])
|
||||
conflict = User.objects.filter(condition).exclude(id=existing_user.id).exists()
|
||||
if conflict:
|
||||
raise AppApiException(500, '用户名、姓名或邮箱已被其他账号占用')
|
||||
with transaction.atomic():
|
||||
existing_user.username = data['username']
|
||||
existing_user.nick_name = data.get('nick_name', '') or existing_user.nick_name
|
||||
if data.get('email'):
|
||||
existing_user.email = data['email']
|
||||
existing_user.phone = data.get('phone', '') or ''
|
||||
existing_user.password = password_encrypt(data['password'])
|
||||
existing_user.is_active = False
|
||||
existing_user.save()
|
||||
# applied_at 是 auto_now_add,只能用 update 直接改库刷新申请时间
|
||||
UserActivationApplication.objects.filter(id=existing_user.activation_application.id).update(
|
||||
status=ActivationStatus.PENDING,
|
||||
reject_reason='',
|
||||
email=existing_user.email or '',
|
||||
email_verified=False,
|
||||
audit_by=None,
|
||||
audit_time=None,
|
||||
applied_at=timezone.now(),
|
||||
)
|
||||
return {'status': ActivationStatus.PENDING, 're_register': True}
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def register(instance):
|
||||
payload = dict(instance)
|
||||
if payload.get('encryptedData'):
|
||||
try:
|
||||
credentials = json.loads(decrypt(payload['encryptedData']))
|
||||
payload['password'] = credentials['password']
|
||||
payload['re_password'] = credentials['re_password']
|
||||
except (ValueError, TypeError, KeyError):
|
||||
raise AppApiException(500, '注册信息无效')
|
||||
serializer = RegisterSerializer(data=payload)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
data = {key: serializer.validated_data.get(key, '') for key in
|
||||
('username', 'nick_name', 'email', 'phone', 'password')}
|
||||
# Public registration never accepts account state, source or role assignments.
|
||||
data.update(pending_activation=True, source='LOCAL')
|
||||
|
||||
existing = User.objects.select_for_update().filter(
|
||||
RegisterSerializer._match_condition(data['username'], data.get('email', ''))
|
||||
).first()
|
||||
if existing is not None:
|
||||
application = UserActivationApplication.objects.select_for_update().filter(user=existing).first()
|
||||
if application is None:
|
||||
raise AppApiException(500, '该用户名或邮箱已存在,请直接登录或联系管理员')
|
||||
if application.status == ActivationStatus.PENDING:
|
||||
raise AppApiException(500, '注册申请审核中,请等待管理员审核,请勿重复提交')
|
||||
if application.status == ActivationStatus.APPROVED:
|
||||
raise AppApiException(500, '该账号已激活,请直接登录')
|
||||
if application.status != ActivationStatus.REJECTED:
|
||||
raise AppApiException(500, '该申请不允许重新提交')
|
||||
# 公开注册入口不能凭用户名或邮箱覆盖别人的被拒绝账号。
|
||||
if (existing.username != data['username'] or existing.source != 'LOCAL'
|
||||
or existing.password != password_encrypt(data['password'])):
|
||||
raise AppApiException(500, '重新提交请使用原用户名和密码')
|
||||
# 仅“已拒绝”的申请允许修改注册信息后重新提交
|
||||
return RegisterSerializer._re_register(existing, data)
|
||||
|
||||
if User.objects.filter(nick_name=data['nick_name']).exists():
|
||||
raise AppApiException(500, '姓名已存在')
|
||||
try:
|
||||
UserManageSerializer().save(data, user_id=None, with_valid=False)
|
||||
except IntegrityError:
|
||||
raise AppApiException(500, '用户名、姓名或邮箱已存在')
|
||||
return {'status': ActivationStatus.PENDING}
|
||||
|
|
@ -0,0 +1,224 @@
|
|||
# coding=utf-8
|
||||
"""用户管理扩展序列化器:待激活审核、角色授权。"""
|
||||
from django.core.cache import cache
|
||||
from django.core.mail import send_mail
|
||||
from django.core.mail.backends.smtp import EmailBackend
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
from rest_framework import serializers
|
||||
|
||||
from common.constants.cache_version import Cache_Version
|
||||
from common.constants.permission_constants import RoleConstants
|
||||
from common.exception.app_exception import AppApiException
|
||||
from common.utils.logger import maxkb_logger
|
||||
from system_manage.models import SystemSetting, SettingType
|
||||
from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment
|
||||
from users.serializers.user import update_user_role
|
||||
|
||||
ROLE_OPTIONS = [
|
||||
{"id": RoleConstants.USER.name, "name": "普通用户"},
|
||||
{"id": RoleConstants.ADMIN.name, "name": "管理员"},
|
||||
]
|
||||
VALID_ROLE_IDS = {item["id"] for item in ROLE_OPTIONS}
|
||||
|
||||
|
||||
def _fmt(value):
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
value = timezone.localtime(value)
|
||||
except Exception:
|
||||
pass
|
||||
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def invalidate_user_auth_cache(user_id):
|
||||
"""角色/状态变化后清掉权限与角色缓存,下一次请求重新计算。"""
|
||||
for cache_version in (Cache_Version.PERMISSION_LIST, Cache_Version.ROLE_LIST, Cache_Version.USER):
|
||||
cache.delete(cache_version.get_key(user_id=user_id), version=cache_version.get_version())
|
||||
|
||||
|
||||
def send_activation_notice(user, approved: bool, reason: str = "") -> bool:
|
||||
"""激活审核结果邮件通知。
|
||||
|
||||
仅在系统设置中配置了发件邮箱(SystemSetting EMAIL)且用户填了邮箱时发送;
|
||||
发送失败只记录日志,不影响审核结果。
|
||||
"""
|
||||
try:
|
||||
if not user.email:
|
||||
return False
|
||||
setting = SystemSetting.objects.filter(type=SettingType.EMAIL.value).first()
|
||||
meta = (setting.meta or {}) if setting else {}
|
||||
if not meta.get("email_host"):
|
||||
return False
|
||||
name = user.nick_name or user.username
|
||||
if approved:
|
||||
subject = "【AI-RAG 知识库平台】账号激活审核通过"
|
||||
body = (f"您好 {name}:\n\n"
|
||||
f"您提交的注册申请已审核通过,账号已激活,现在可以登录平台使用。\n\n"
|
||||
f"如非本人操作,请忽略本邮件。")
|
||||
else:
|
||||
subject = "【AI-RAG 知识库平台】账号激活审核未通过"
|
||||
body = (f"您好 {name}:\n\n"
|
||||
f"很抱歉,您提交的注册申请未通过审核。\n"
|
||||
f"拒绝原因:{reason or '不符合激活条件'}\n\n"
|
||||
f"请在注册页面使用原用户名和密码修改并重新提交资料,审核通过后即可登录。")
|
||||
connection = EmailBackend(meta.get("email_host"), meta.get("email_port"),
|
||||
meta.get("email_host_user"), meta.get("email_host_password"),
|
||||
meta.get("email_use_tls"), False, meta.get("email_use_ssl"), timeout=10)
|
||||
send_mail(subject, body,
|
||||
from_email=meta.get("from_email") or meta.get("email_host_user"),
|
||||
recipient_list=[user.email],
|
||||
fail_silently=False, connection=connection)
|
||||
return True
|
||||
except Exception as exc:
|
||||
maxkb_logger.error(f"send activation notice failed: user={getattr(user, 'id', '')} {exc}")
|
||||
return False
|
||||
|
||||
|
||||
class ActivationSerializer(serializers.Serializer):
|
||||
keyword = serializers.CharField(required=False, allow_blank=True, default="")
|
||||
status = serializers.CharField(required=False, allow_blank=True, default=ActivationStatus.PENDING)
|
||||
|
||||
@staticmethod
|
||||
def page(current_page: int, page_size: int, keyword: str = "", status: str = ActivationStatus.PENDING):
|
||||
query_set = UserActivationApplication.objects.select_related("user").all()
|
||||
if status == 'UNAPPROVED':
|
||||
query_set = query_set.filter(status__in=[ActivationStatus.PENDING, ActivationStatus.REJECTED])
|
||||
elif status and status != "ALL":
|
||||
query_set = query_set.filter(status=status)
|
||||
if keyword:
|
||||
query_set = query_set.filter(
|
||||
Q(user__username__icontains=keyword)
|
||||
| Q(user__nick_name__icontains=keyword)
|
||||
| Q(user__email__icontains=keyword)
|
||||
| Q(user__phone__icontains=keyword)
|
||||
)
|
||||
total = query_set.count()
|
||||
start = max(0, (current_page - 1) * page_size)
|
||||
records = []
|
||||
for application in query_set.order_by("-applied_at")[start:start + page_size]:
|
||||
user = application.user
|
||||
records.append({
|
||||
"id": str(application.id),
|
||||
"user_id": str(user.id),
|
||||
"username": user.username,
|
||||
"nick_name": user.nick_name,
|
||||
"email": user.email or application.email or "",
|
||||
"phone": user.phone or '',
|
||||
"email_verified": application.email_verified,
|
||||
"status": application.status,
|
||||
"applied_at": _fmt(application.applied_at),
|
||||
"audit_time": _fmt(application.audit_time),
|
||||
"reject_reason": application.reject_reason,
|
||||
})
|
||||
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def approve(application_id: str, audit_user_id: str, role_id: str = RoleConstants.USER.name,
|
||||
workspace_id: str = "default"):
|
||||
application = (UserActivationApplication.objects.select_for_update()
|
||||
.select_related("user").filter(id=application_id).first())
|
||||
if application is None:
|
||||
raise AppApiException(500, "待激活申请不存在")
|
||||
if application.status != ActivationStatus.PENDING:
|
||||
raise AppApiException(500, "该申请已处理,不能重复审核")
|
||||
if role_id not in VALID_ROLE_IDS:
|
||||
raise AppApiException(500, "角色不合法")
|
||||
application.status = ActivationStatus.APPROVED
|
||||
application.reject_reason = ""
|
||||
application.audit_by_id = audit_user_id
|
||||
application.audit_time = timezone.now()
|
||||
application.save()
|
||||
|
||||
user = application.user
|
||||
user.is_active = True
|
||||
user.role = role_id
|
||||
user.save()
|
||||
|
||||
# 审核通过时同步写入角色授权关系,形成独立的授权数据对象
|
||||
UserRoleAssignment.objects.filter(user_id=user.id).delete()
|
||||
UserRoleAssignment.objects.create(
|
||||
user_id=user.id, role_id=role_id,
|
||||
workspace_id=workspace_id or "default", granted_by_id=audit_user_id
|
||||
)
|
||||
invalidate_user_auth_cache(user.id)
|
||||
transaction.on_commit(lambda: send_activation_notice(user, True))
|
||||
return {"id": str(application.id), "status": application.status, "user_id": str(user.id)}
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def reject(application_id: str, audit_user_id: str, reason: str = ""):
|
||||
application = (UserActivationApplication.objects.select_for_update()
|
||||
.select_related("user").filter(id=application_id).first())
|
||||
if application is None:
|
||||
raise AppApiException(500, "待激活申请不存在")
|
||||
if application.status != ActivationStatus.PENDING:
|
||||
raise AppApiException(500, "该申请已处理,不能重复审核")
|
||||
application.status = ActivationStatus.REJECTED
|
||||
application.audit_by_id = audit_user_id
|
||||
application.audit_time = timezone.now()
|
||||
application.reject_reason = reason or "不符合激活条件"
|
||||
application.save()
|
||||
application.user.is_active = False
|
||||
application.user.save()
|
||||
invalidate_user_auth_cache(application.user.id)
|
||||
transaction.on_commit(lambda: send_activation_notice(application.user, False, application.reject_reason))
|
||||
return {"id": str(application.id), "status": application.status, "user_id": str(application.user_id)}
|
||||
|
||||
|
||||
class RoleAssignmentSerializer(serializers.Serializer):
|
||||
role_id = serializers.CharField(required=True)
|
||||
workspace_ids = serializers.ListField(child=serializers.CharField(), required=False, allow_empty=True)
|
||||
|
||||
@staticmethod
|
||||
def detail(user_id: str):
|
||||
user = User.objects.filter(id=user_id).first()
|
||||
if user is None:
|
||||
raise AppApiException(500, "用户不存在")
|
||||
assignments = UserRoleAssignment.objects.filter(user_id=user_id).order_by("granted_at")
|
||||
records = [{
|
||||
"role_id": item.role_id,
|
||||
"workspace_id": item.workspace_id,
|
||||
"granted_at": _fmt(item.granted_at),
|
||||
} for item in assignments]
|
||||
if not records:
|
||||
records = [{"role_id": user.role, "workspace_id": "default", "granted_at": ""}]
|
||||
return {
|
||||
"user": {"id": str(user.id), "username": user.username, "nick_name": user.nick_name,
|
||||
"email": user.email or "", "role": user.role, "is_active": user.is_active},
|
||||
"assignments": records,
|
||||
"available_roles": ROLE_OPTIONS,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@transaction.atomic
|
||||
def save(user_id: str, role_settings, granted_by_id: str):
|
||||
user = User.objects.select_for_update().filter(id=user_id).first()
|
||||
if user is None:
|
||||
raise AppApiException(500, "用户不存在")
|
||||
if not isinstance(role_settings, list) or len(role_settings) != 1 or not isinstance(role_settings[0], dict):
|
||||
raise AppApiException(500, "请选择一个角色")
|
||||
item = role_settings[0]
|
||||
role_id = item.get("role_id")
|
||||
if role_id not in VALID_ROLE_IDS:
|
||||
raise AppApiException(500, "角色只能为普通用户或管理员")
|
||||
if item.get('workspace_ids') not in (None, [], ['default']):
|
||||
raise AppApiException(500, '角色配置无效')
|
||||
if str(user.id) == 'f0dd8f71-e4ee-11ee-8c84-a8a1595801ab':
|
||||
if role_id != RoleConstants.ADMIN.name:
|
||||
raise AppApiException(500, '不能修改内置管理员角色')
|
||||
return RoleAssignmentSerializer.detail(str(user.id))
|
||||
UserRoleAssignment.objects.filter(user_id=user.id).delete()
|
||||
UserRoleAssignment.objects.create(
|
||||
user_id=user.id, role_id=role_id, workspace_id='default', granted_by_id=granted_by_id
|
||||
)
|
||||
user.role = role_id
|
||||
user.save()
|
||||
update_user_role({'role_setting': [{'role_id': role_id, 'workspace_ids': [
|
||||
'None' if role_id == RoleConstants.ADMIN.name else 'default'
|
||||
]}]}, user, granted_by_id)
|
||||
invalidate_user_auth_cache(user.id)
|
||||
return RoleAssignmentSerializer.detail(str(user.id))
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework.permissions import AllowAny
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from common.result import result
|
||||
from users.serializers.register import RegisterSerializer
|
||||
|
||||
|
||||
class RegisterView(APIView):
|
||||
authentication_classes = []
|
||||
permission_classes = [AllowAny]
|
||||
|
||||
@extend_schema(summary='用户注册', tags=['User Management'])
|
||||
def post(self, request):
|
||||
return result.success(RegisterSerializer.register(request.data))
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
# coding=utf-8
|
||||
"""用户管理扩展视图:待激活审核、角色授权。"""
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from common.auth.authenticate import TokenAuth
|
||||
from common.auth.authentication import has_permissions
|
||||
from common.constants.permission_constants import PermissionConstants, RoleConstants
|
||||
from common.log.log import log
|
||||
from common.result import result
|
||||
from users.models import ActivationStatus
|
||||
from users.serializers.user_manage_ext import ActivationSerializer, RoleAssignmentSerializer
|
||||
|
||||
|
||||
class UserActivationPendingView(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@extend_schema(methods=['GET'], summary="待激活注册申请列表", tags=["User Management"])
|
||||
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
|
||||
def get(self, request: Request):
|
||||
current_page = int(request.query_params.get("current_page", 1))
|
||||
page_size = int(request.query_params.get("page_size", 20))
|
||||
keyword = request.query_params.get("keyword", "")
|
||||
status = request.query_params.get("status", ActivationStatus.PENDING)
|
||||
return result.success(ActivationSerializer.page(current_page, page_size, keyword, status))
|
||||
|
||||
|
||||
class UserActivationApproveView(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@extend_schema(methods=['POST'], summary="通过待激活注册申请", tags=["User Management"])
|
||||
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
|
||||
@log(menu='User management', operate='Approve user activation',
|
||||
get_operation_object=lambda r, k: {'name': k.get('application_id')})
|
||||
def post(self, request: Request, application_id: str):
|
||||
role_id = request.data.get("role_id", RoleConstants.USER.name)
|
||||
workspace_id = request.data.get("workspace_id", "default")
|
||||
return result.success(
|
||||
ActivationSerializer.approve(application_id, str(request.user.id), role_id, workspace_id)
|
||||
)
|
||||
|
||||
|
||||
class UserActivationRejectView(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@extend_schema(methods=['POST'], summary="拒绝待激活注册申请", tags=["User Management"])
|
||||
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
|
||||
@log(menu='User management', operate='Reject user activation',
|
||||
get_operation_object=lambda r, k: {'name': k.get('application_id')})
|
||||
def post(self, request: Request, application_id: str):
|
||||
reason = request.data.get("reason", "")
|
||||
return result.success(
|
||||
ActivationSerializer.reject(application_id, str(request.user.id), reason)
|
||||
)
|
||||
|
||||
|
||||
class UserRoleAssignmentView(APIView):
|
||||
authentication_classes = [TokenAuth]
|
||||
|
||||
@extend_schema(methods=['GET'], summary="查看用户角色授权", tags=["User Management"])
|
||||
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
|
||||
def get(self, request: Request, user_id: str):
|
||||
return result.success(RoleAssignmentSerializer.detail(user_id))
|
||||
|
||||
@extend_schema(methods=['PUT'], summary="分配用户角色", tags=["User Management"])
|
||||
@has_permissions(PermissionConstants.USER_EDIT, RoleConstants.ADMIN)
|
||||
@log(menu='User management', operate='Assign user role',
|
||||
get_operation_object=lambda r, k: {'name': k.get('user_id')})
|
||||
def put(self, request: Request, user_id: str):
|
||||
return result.success(
|
||||
RoleAssignmentSerializer.save(user_id, request.data.get("role_settings", []), str(request.user.id))
|
||||
)
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# 思维导图图片资源
|
||||
|
||||
解析任务、手工提交和保存图谱时,将支持的 Base64 位图转换为私有图片文件,Markdown 保存稳定的 `/admin/api/workspace/.../ai_hub/assets/.../content` 地址。历史图谱在再次保存时转换原稿和历史版本中的内联图片。
|
||||
|
||||
默认持久化目录为项目根目录 `data/ai-hub-assets`;可通过环境变量 `AI_HUB_ASSET_DIR` 指定。Web 与独立 Hub Worker 必须使用相同目录,多机部署需挂载共享存储。备份应同时包含现有数据库与此目录。增量发布不可覆盖该目录。
|
||||
|
||||
文件以工作空间、用户和内容哈希去重,旁路 JSON 保存归属、格式、大小和最后引用时间。无需数据库迁移。图片接口使用现有用户和工作空间鉴权,父页面通过二进制通信供导图预览使用,令牌不进入图片 URL。浏览器仅在当前页面内缓存图片,图片展示可点击放大,失败后可重试。
|
||||
|
||||
单图上限 20 MiB、4000 万像素;单次转换图片合计上限 100 MiB。支持 PNG、JPEG、GIF、WebP、BMP。图谱及其原稿、历史版本所引用的图片长期保留;运行中或待处理任务中的图片不会清理。其余资源最后引用七天后可清理,未保存的旧任务图片届时可能失效。
|
||||
|
||||
已有独立 Hub Worker 每运行 24 小时执行一次清理,不新增定时进程。手工核验可在原有 Web 环境中执行:
|
||||
|
||||
```sh
|
||||
venv/bin/python scripts/ai_hub_ops.py cleanup-assets
|
||||
venv/bin/python scripts/ai_hub_ops.py cleanup-assets --apply
|
||||
```
|
||||
|
||||
首条命令只报告数量。清理范围仅限本模块资源目录,不涉及原文档、数据库记录、模型或解析服务文件。
|
||||
|
||||
含资源图片的 Markdown 下载为 ZIP,包含 `document.md` 和 `images/`;HTML、导图 SVG 导出会嵌入图片,以便离线打开。智能整理使用图片占位符;重复占位符只恢复一次,遗漏的图片自动补入“原文图片”部分,解析原稿始终保留。
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
"""Explicit AI hub operations; defaults to a read-only schema check, never auto-migrates."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['check', 'plan', 'migrate', 'worker', 'cleanup-assets'], nargs='?', default='check')
|
||||
parser.add_argument('--apply', action='store_true', help='Apply expired image cleanup (default: report only)')
|
||||
parser.add_argument('--once', action='store_true')
|
||||
parser.add_argument('--recover', action='store_true')
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.chdir(root)
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ['SERVER_NAME'] = 'web'
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
from django.conf import settings
|
||||
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
|
||||
settings.LANGUAGE_CODE = 'zh-hans'
|
||||
import django
|
||||
django.setup()
|
||||
from django.core.management import call_command
|
||||
from django.db import connection
|
||||
from django.db.migrations.executor import MigrationExecutor
|
||||
target = [('system_manage', '0007_ai_hub')]
|
||||
if args.action in ('plan', 'migrate'):
|
||||
executor = MigrationExecutor(connection)
|
||||
plan = executor.migration_plan(target)
|
||||
if any(backwards for _, backwards in plan):
|
||||
raise RuntimeError('存在更新迁移,不允许回退,请先核对版本')
|
||||
call_command('migrate', 'system_manage', '0007_ai_hub', plan=args.action == 'plan', interactive=False)
|
||||
elif args.action == 'cleanup-assets':
|
||||
call_command('ai_hub_cleanup_assets', apply=args.apply)
|
||||
elif args.action == 'worker':
|
||||
call_command('ai_hub_worker', once=args.once, recover=args.recover)
|
||||
else:
|
||||
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
|
||||
tables = set(connection.introspection.table_names())
|
||||
for model in (AIParserResource, AIFlow, AIService, AITask, AIArtifact):
|
||||
print(f'{model._meta.db_table}: {"ready" if model._meta.db_table in tables else "missing"}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
"""Isolated SQLite verification; never connects to the shared PostgreSQL or Redis."""
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
settings.LOGGING_CONFIG = None
|
||||
settings.USE_I18N = False
|
||||
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
|
||||
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
|
||||
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
|
||||
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
|
||||
'system_manage', 'models_provider', 'application', 'chat', 'oss')]
|
||||
import django
|
||||
django.setup()
|
||||
if '--migration' in sys.argv or '--migration-check' in sys.argv:
|
||||
from django.core.management import call_command
|
||||
call_command('makemigrations', 'system_manage', name='ai_hub',
|
||||
check='--migration-check' in sys.argv, dry_run='--migration-check' in sys.argv)
|
||||
sys.exit(0)
|
||||
from django.db import connection
|
||||
from users.models import User
|
||||
from users.models.user_manage_ext import UserActivationApplication
|
||||
from models_provider.models import Model
|
||||
from knowledge.models import KnowledgeFolder, Knowledge
|
||||
from system_manage.models.ai_hub import AIParserResource, AIFlow, AIService, AITask, AIArtifact
|
||||
with connection.schema_editor() as editor:
|
||||
for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge, AIParserResource, AIFlow, AIService, AITask, AIArtifact):
|
||||
editor.create_model(model)
|
||||
suite = unittest.defaultTestLoader.loadTestsFromName('system_manage.test_ai_hub')
|
||||
outcome = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
sys.exit(0 if outcome.wasSuccessful() else 1)
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
"""Run knowledge metadata and segmentation regression tests against an isolated in-memory database."""
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
settings.LOGGING_CONFIG = None
|
||||
settings.USE_I18N = False
|
||||
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
|
||||
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
|
||||
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
|
||||
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
|
||||
'system_manage', 'models_provider', 'application', 'chat', 'oss',
|
||||
)]
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from django.db import connection
|
||||
from users.models import User, UserActivationApplication
|
||||
from models_provider.models import Model
|
||||
from knowledge.models import KnowledgeFolder, Knowledge
|
||||
with connection.schema_editor() as editor:
|
||||
for model in (User, UserActivationApplication, Model, KnowledgeFolder, Knowledge):
|
||||
editor.create_model(model)
|
||||
suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_knowledge_ext')
|
||||
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
sys.exit(0 if result.wasSuccessful() else 1)
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
"""Run independent index rebuild regression tests against an isolated in-memory database."""
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
settings.LOGGING_CONFIG = None
|
||||
settings.USE_I18N = False
|
||||
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
|
||||
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
|
||||
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
|
||||
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
|
||||
'system_manage', 'models_provider', 'application', 'chat', 'oss',
|
||||
)]
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from django.db import connection
|
||||
from knowledge.models import KnowledgeRebuildTask
|
||||
|
||||
with connection.schema_editor() as editor:
|
||||
editor.create_model(KnowledgeRebuildTask)
|
||||
|
||||
suite = unittest.defaultTestLoader.loadTestsFromName('knowledge.test_rebuild')
|
||||
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
sys.exit(0 if result.wasSuccessful() else 1)
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
"""Exercise the signed API under the actual minimal model-service app configuration."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'apps'))
|
||||
os.environ['SERVER_NAME'] = 'local_model'
|
||||
os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0'
|
||||
from django.conf import settings
|
||||
settings.configure(SECRET_KEY='test-only', AUTH_HANDLES=[], USE_I18N=False,
|
||||
INSTALLED_APPS=['django.contrib.contenttypes', 'local_model'],
|
||||
DATABASES={'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}},
|
||||
CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}})
|
||||
import django
|
||||
django.setup()
|
||||
from rest_framework.test import APIRequestFactory
|
||||
from common.model_runtime_client import signed_request
|
||||
from local_model.views.runtime import LocalModelRuntime
|
||||
factory = APIRequestFactory()
|
||||
with patch('common.model_runtime_client._secret', return_value=b'test-only'), patch('local_model.views.runtime.manager') as manager:
|
||||
manager.status.return_value = {'status': 'UNLOADED'}
|
||||
body, signature = signed_request({'operation': 'status', 'model_id': '00000000-0000-4000-8000-000000000001'})
|
||||
response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body,
|
||||
content_type='application/json', HTTP_X_MODEL_RUNTIME_SIGNATURE=signature))
|
||||
assert response.status_code == 200, response.content
|
||||
manager.status.assert_called_once()
|
||||
manager.reset_mock()
|
||||
response = LocalModelRuntime.as_view()(factory.post('/model_runtime', data=body, content_type='application/json'))
|
||||
assert response.status_code == 403, response.status_code
|
||||
manager.status.assert_not_called()
|
||||
print('Minimal model-service API: signed request accepted; unsigned request rejected')
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
"""Model loading regressions against SQLite in memory; never connects to shared services."""
|
||||
from pathlib import Path
|
||||
|
||||
source = (Path(__file__).with_name('check_knowledge_ext.py')).read_text(encoding='utf-8')
|
||||
source = source.replace("loadTestsFromName('knowledge.test_knowledge_ext')", "loadTestsFromNames(['models_provider.test_model_runtime', 'models_provider.test_model_loading_api', 'knowledge.test_knowledge_ext'])")
|
||||
exec(compile(source, __file__, 'exec'))
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
"""Run account registration regression tests against an isolated in-memory database."""
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
settings.LOGGING_CONFIG = None
|
||||
settings.USE_I18N = False
|
||||
settings.CACHES = {'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}
|
||||
settings.DATABASES = {'default': {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}}
|
||||
settings.INSTALLED_APPS = [AppConfig(name, importlib.import_module(name)) for name in (
|
||||
'django.contrib.contenttypes', 'users', 'tools', 'knowledge', 'common',
|
||||
'system_manage', 'models_provider', 'application', 'chat', 'oss',
|
||||
)]
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from django.db import connection
|
||||
from users.models import User, UserActivationApplication, UserRoleAssignment
|
||||
|
||||
with connection.schema_editor() as editor:
|
||||
for model in (User, UserActivationApplication, UserRoleAssignment):
|
||||
editor.create_model(model)
|
||||
|
||||
suite = unittest.defaultTestLoader.loadTestsFromName('users.tests.RegistrationTests')
|
||||
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
sys.exit(0 if result.wasSuccessful() else 1)
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
"""Single-process development web server; never migrates the shared database."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--host', default='127.0.0.1')
|
||||
parser.add_argument('--port', type=int, default=8080)
|
||||
parser.add_argument('--check', action='store_true', help='Check Django configuration without starting the server')
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.chdir(root)
|
||||
sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ['SERVER_NAME'] = 'web'
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
|
||||
# Redirect local logs before Django imports its logging configuration.
|
||||
from maxkb import const
|
||||
const.LOG_DIR = str(root / 'data' / 'dev' / 'logs')
|
||||
from django.conf import settings
|
||||
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
|
||||
settings.LANGUAGE_CODE = 'zh-hans'
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
from django.core import management
|
||||
|
||||
if args.check:
|
||||
management.call_command('check')
|
||||
else:
|
||||
# No autoreloader: avoid loading a second copy of the Python application.
|
||||
management.call_command('runserver', f'{args.host}:{args.port}', use_reloader=False, insecure_serving=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
"""Single-process weight service with startup preloads; never runs migrations."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--host')
|
||||
parser.add_argument('--port', type=int)
|
||||
parser.add_argument('--check', action='store_true')
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.chdir(root); sys.path.insert(0, str(root / 'apps'))
|
||||
os.environ['SERVER_NAME'] = 'local_model'
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
|
||||
if args.check:
|
||||
os.environ['MAXKB_RUNTIME_AUTOSTART'] = '0'
|
||||
from maxkb.const import CONFIG
|
||||
from django.conf import settings
|
||||
if settings.LANGUAGE_CODE.lower() == 'zh-cn':
|
||||
settings.LANGUAGE_CODE = 'zh-hans'
|
||||
import django
|
||||
django.setup()
|
||||
from django.core import management
|
||||
if args.check:
|
||||
management.call_command('check')
|
||||
else:
|
||||
bind = f"{args.host or CONFIG.get('LOCAL_MODEL_HOST')}:{args.port or CONFIG.get('LOCAL_MODEL_PORT')}"
|
||||
management.call_command('runserver', bind, use_reloader=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
import { get, post, request } from '@/request/index'
|
||||
import useStore from '@/stores'
|
||||
|
||||
export const hubBase = () => `/workspace/${useStore().user.getWorkspaceId()}/ai_hub`
|
||||
export const hubGet = (kind: string, id?: string) => get(`${hubBase()}/${kind}${id ? '/' + id : ''}`)
|
||||
export const hubPost = (kind: string, data: any, id?: string) => post(`${hubBase()}/${kind}${id ? '/' + id : ''}`, data)
|
||||
export async function hubBinary(kind: 'assets' | 'exports', id?: string, data?: any) {
|
||||
const response = await request({ url: `${hubBase()}/${kind}${id ? '/' + id + '/content' : ''}`,
|
||||
method: kind === 'assets' ? 'get' : 'post', data, responseType: 'blob' })
|
||||
if (!(response.data instanceof Blob) || response.data.type.includes('json')) throw new Error('图片或导出文件不可用')
|
||||
return { content: await response.data.arrayBuffer(), mime: response.data.type }
|
||||
}
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
<template>
|
||||
<el-drawer v-model="visible" size="560px" title="索引重建任务" :destroy-on-close="false">
|
||||
<div v-loading="loading" class="rebuild-detail">
|
||||
<div class="rebuild-head">
|
||||
<div>
|
||||
<div class="rebuild-title">任务 {{ shortId }}</div>
|
||||
<div class="rebuild-sub">{{ task.config_snapshot?.knowledge_name || '知识库' }}</div>
|
||||
</div>
|
||||
<el-tag :type="statusType" effect="light">{{ task.status_label || task.status || '-' }}</el-tag>
|
||||
</div>
|
||||
|
||||
<el-progress
|
||||
:percentage="task.progress || 0"
|
||||
:status="progressStatus"
|
||||
:stroke-width="10"
|
||||
class="rebuild-progress"
|
||||
/>
|
||||
|
||||
<el-descriptions :column="2" border size="small" class="rebuild-desc">
|
||||
<el-descriptions-item label="文档总数">{{ task.document_total ?? 0 }}</el-descriptions-item>
|
||||
<el-descriptions-item label="成功 / 失败">
|
||||
<span class="color-success">{{ task.document_success ?? 0 }}</span>
|
||||
/
|
||||
<span class="color-danger">{{ task.document_failed ?? 0 }}</span>
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="向量模型">
|
||||
{{ task.config_snapshot?.embedding_model_name || task.config_snapshot?.embedding_model || '-' }}
|
||||
</el-descriptions-item>
|
||||
<el-descriptions-item label="任务 ID">{{ shortId }}</el-descriptions-item>
|
||||
<el-descriptions-item label="提交时间">{{ task.created_at || '-' }}</el-descriptions-item>
|
||||
<el-descriptions-item label="结束时间">{{ task.finished_at || '-' }}</el-descriptions-item>
|
||||
</el-descriptions>
|
||||
|
||||
<el-alert
|
||||
v-if="task.error_message"
|
||||
:title="task.error_message"
|
||||
:type="task.status === 'FAILED' ? 'error' : 'warning'"
|
||||
:closable="false"
|
||||
show-icon
|
||||
class="rebuild-error"
|
||||
/>
|
||||
|
||||
<div class="rebuild-tip">
|
||||
重建期间检索可能暂不可用。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<template #footer>
|
||||
<el-button @click="visible = false">{{ $t('common.cancel') }}</el-button>
|
||||
<el-button
|
||||
v-if="task.status === 'FAILED' || task.status === 'PARTIAL'"
|
||||
type="primary"
|
||||
:loading="loading"
|
||||
@click="retry"
|
||||
>
|
||||
重新发起
|
||||
</el-button>
|
||||
</template>
|
||||
</el-drawer>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, onUnmounted, reactive, ref, watch } from 'vue'
|
||||
import knowledgeApi from '@/api/knowledge/knowledge'
|
||||
import { MsgSuccess } from '@/utils/message'
|
||||
import { t } from '@/locales'
|
||||
|
||||
const emit = defineEmits(['refresh'])
|
||||
const visible = ref(false)
|
||||
const loading = ref(false)
|
||||
const knowledgeId = ref('')
|
||||
const taskId = ref('')
|
||||
const task = reactive<any>({ config_snapshot: {} })
|
||||
let timer: any = null
|
||||
|
||||
const shortId = computed(() => (task.id ? String(task.id).slice(0, 8) : '-'))
|
||||
const statusType = computed(() => {
|
||||
const map: Record<string, string> = {
|
||||
PENDING: 'info',
|
||||
RUNNING: 'warning',
|
||||
SUCCESS: 'success',
|
||||
PARTIAL: 'warning',
|
||||
FAILED: 'danger',
|
||||
}
|
||||
return map[task.status] || 'info'
|
||||
})
|
||||
const progressStatus = computed(() => {
|
||||
if (task.status === 'SUCCESS') return 'success'
|
||||
if (task.status === 'FAILED') return 'exception'
|
||||
return undefined
|
||||
})
|
||||
|
||||
function stopPolling() {
|
||||
if (timer) {
|
||||
clearInterval(timer)
|
||||
timer = null
|
||||
}
|
||||
}
|
||||
|
||||
function loadTask() {
|
||||
if (!visible.value || !knowledgeId.value || !taskId.value) return
|
||||
knowledgeApi.getRebuildTask(knowledgeId.value, taskId.value, loading).then((res: any) => {
|
||||
Object.assign(task, res.data || {})
|
||||
if (['SUCCESS', 'FAILED', 'PARTIAL'].includes(task.status)) {
|
||||
stopPolling()
|
||||
emit('refresh')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function startPolling() {
|
||||
stopPolling()
|
||||
timer = setInterval(loadTask, 2000)
|
||||
}
|
||||
|
||||
function open(id: string, kid: string) {
|
||||
knowledgeId.value = kid
|
||||
taskId.value = id
|
||||
visible.value = true
|
||||
Object.keys(task).forEach((key) => delete task[key])
|
||||
startPolling()
|
||||
loadTask()
|
||||
}
|
||||
|
||||
function retry() {
|
||||
knowledgeApi.postRetryRebuildTask(knowledgeId.value, taskId.value, loading).then((res: any) => {
|
||||
const next = res.data
|
||||
MsgSuccess('已重新发起重建任务')
|
||||
taskId.value = next.id
|
||||
Object.assign(task, next)
|
||||
startPolling()
|
||||
})
|
||||
}
|
||||
|
||||
watch(visible, (value) => { if (!value) stopPolling() })
|
||||
onUnmounted(stopPolling)
|
||||
defineExpose({ open })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.rebuild-detail {
|
||||
min-height: 260px;
|
||||
}
|
||||
.rebuild-head {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.rebuild-title {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
color: #1f2329;
|
||||
}
|
||||
.rebuild-sub {
|
||||
font-size: 12.5px;
|
||||
color: #646a73;
|
||||
margin-top: 4px;
|
||||
}
|
||||
.rebuild-progress {
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
.rebuild-desc {
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
.rebuild-error {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.rebuild-tip {
|
||||
font-size: 12px;
|
||||
color: #8f959e;
|
||||
line-height: 1.7;
|
||||
}
|
||||
.color-success {
|
||||
color: #1a9c62;
|
||||
font-weight: 600;
|
||||
}
|
||||
.color-danger {
|
||||
color: #c9353f;
|
||||
font-weight: 600;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
<template>
|
||||
<LoginLayout>
|
||||
<LoginContainer>
|
||||
<template #logo>
|
||||
<div class="login-brand">
|
||||
<ChinaMobileIcon />
|
||||
<h1 class="login-brand-title">AI-RAG</h1>
|
||||
</div>
|
||||
</template>
|
||||
<h2 class="mb-24">用户注册</h2>
|
||||
<el-form
|
||||
ref="formRef"
|
||||
:model="form"
|
||||
:rules="rules"
|
||||
:disabled="submitting"
|
||||
@submit.prevent="submit"
|
||||
>
|
||||
<el-form-item prop="username">
|
||||
<el-input v-model="form.username" size="large" placeholder="用户名" aria-label="用户名" maxlength="64" autocomplete="username" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="nick_name">
|
||||
<el-input v-model="form.nick_name" size="large" placeholder="姓名" aria-label="姓名" maxlength="64" autocomplete="name" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="email">
|
||||
<el-input v-model="form.email" size="large" placeholder="邮箱" aria-label="邮箱" autocomplete="email" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="phone">
|
||||
<el-input v-model="form.phone" size="large" placeholder="手机号(选填)" aria-label="手机号" maxlength="20" autocomplete="tel" />
|
||||
</el-form-item>
|
||||
<el-form-item prop="password">
|
||||
<el-input v-model="form.password" type="password" size="large" placeholder="密码" aria-label="密码" maxlength="20" autocomplete="new-password" show-password />
|
||||
</el-form-item>
|
||||
<el-form-item prop="re_password">
|
||||
<el-input v-model="form.re_password" type="password" size="large" placeholder="确认密码" aria-label="确认密码" maxlength="20" autocomplete="new-password" show-password />
|
||||
</el-form-item>
|
||||
<el-button native-type="submit" type="primary" size="large" class="w-full" :loading="submitting" :disabled="initializing">
|
||||
注册
|
||||
</el-button>
|
||||
</el-form>
|
||||
<div class="mt-12">
|
||||
<el-button link type="primary" @click="router.push('/login')">返回登录</el-button>
|
||||
</div>
|
||||
</LoginContainer>
|
||||
</LoginLayout>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { onBeforeMount, reactive, ref } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import type { FormInstance, FormRules } from 'element-plus'
|
||||
import forge from 'node-forge'
|
||||
import LoginLayout from '@/layout/login-layout/LoginLayout.vue'
|
||||
import LoginContainer from '@/layout/login-layout/LoginContainer.vue'
|
||||
import ChinaMobileIcon from '@/components/china-mobile-icon/index.vue'
|
||||
import UserApi from '@/api/user/user'
|
||||
import useStore from '@/stores'
|
||||
import { MsgError, MsgSuccess } from '@/utils/message'
|
||||
|
||||
const router = useRouter()
|
||||
const { user } = useStore()
|
||||
const formRef = ref<FormInstance>()
|
||||
const initializing = ref(true)
|
||||
const submitting = ref(false)
|
||||
const form = reactive({ username: '', nick_name: '', email: '', phone: '', password: '', re_password: '' })
|
||||
const rules: FormRules<typeof form> = {
|
||||
username: [
|
||||
{ required: true, message: '请输入用户名', trigger: 'blur' },
|
||||
{ min: 4, max: 64, message: '用户名为 4–64 个字符', trigger: 'blur' },
|
||||
],
|
||||
nick_name: [{ required: true, whitespace: true, message: '请输入姓名', trigger: 'blur' }],
|
||||
email: [
|
||||
{ required: true, message: '请输入邮箱', trigger: 'blur' },
|
||||
{ type: 'email', message: '邮箱格式不正确', trigger: 'blur' },
|
||||
],
|
||||
phone: [{ pattern: /^1[3-9]\d{9}$/, message: '手机号格式不正确', trigger: 'blur' }],
|
||||
password: [
|
||||
{ required: true, message: '请输入密码', trigger: 'blur' },
|
||||
{
|
||||
pattern: /^(?=.*[a-z])(?=.*[-_!@#$%^&*`~.()+=])(?:(?=.*[A-Z])|(?=.*\d))[a-zA-Z0-9\-_!@#$%^&*`~.()+=]{6,20}$/,
|
||||
message: '密码为 6–20 位,含小写字母、特殊字符及数字或大写字母',
|
||||
trigger: 'blur',
|
||||
},
|
||||
],
|
||||
re_password: [
|
||||
{ required: true, message: '请确认密码', trigger: 'blur' },
|
||||
{
|
||||
validator: (_rule, value, callback) => {
|
||||
callback(value === form.password ? undefined : new Error('两次密码不一致'))
|
||||
},
|
||||
trigger: 'blur',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
onBeforeMount(async () => {
|
||||
try {
|
||||
await user.asyncGetProfile()
|
||||
} catch {
|
||||
MsgError('加载失败,请刷新重试')
|
||||
} finally {
|
||||
initializing.value = false
|
||||
}
|
||||
})
|
||||
|
||||
async function submit() {
|
||||
if (submitting.value || initializing.value) return
|
||||
const valid = await formRef.value?.validate().catch(() => false)
|
||||
if (!valid) return
|
||||
submitting.value = true
|
||||
try {
|
||||
if (!user.rasKey) await user.asyncGetProfile()
|
||||
const key = forge.pki.publicKeyFromPem(user.rasKey)
|
||||
const credentials = JSON.stringify({ password: form.password, re_password: form.re_password })
|
||||
const encryptedData = forge.util.encode64(key.encrypt(forge.util.encodeUtf8(credentials), 'RSAES-PKCS1-V1_5'))
|
||||
await UserApi.register({
|
||||
username: form.username.trim(),
|
||||
nick_name: form.nick_name.trim(),
|
||||
email: form.email.trim(),
|
||||
phone: form.phone.trim(),
|
||||
encryptedData,
|
||||
})
|
||||
form.password = ''
|
||||
form.re_password = ''
|
||||
MsgSuccess('注册成功,待管理员审核')
|
||||
await router.replace('/login')
|
||||
} catch {
|
||||
// Request errors are displayed by the shared response handler.
|
||||
} finally {
|
||||
submitting.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.login-brand {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 48px;
|
||||
:deep(.china-mobile-icon) {
|
||||
--china-mobile-icon-width: 236px;
|
||||
--china-mobile-icon-height: 30px;
|
||||
--china-mobile-icon-gap: 16px;
|
||||
}
|
||||
}
|
||||
.login-brand-title {
|
||||
margin: 0;
|
||||
font-size: 24px;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.8px;
|
||||
background: linear-gradient(90deg, #6b47e5, #1e71c7);
|
||||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
}
|
||||
:deep(.right-container) {
|
||||
overflow-y: auto;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
<template>
|
||||
<div class="hub-manager" v-loading="loading">
|
||||
<el-tabs v-model="tab" @tab-change="load">
|
||||
<el-tab-pane label="服务" name="services" />
|
||||
<el-tab-pane label="流程" name="flows" />
|
||||
<el-tab-pane label="任务" name="tasks" />
|
||||
<el-tab-pane label="图谱" name="artifacts" />
|
||||
</el-tabs>
|
||||
<div class="mb-16">
|
||||
<el-button v-if="tab === 'flows' || (resources.is_admin && tab === 'services')" type="primary" @click="create">{{ tab === 'services' ? '发布服务' : '新建流程' }}</el-button>
|
||||
<el-button @click="load">刷新</el-button>
|
||||
</div>
|
||||
<el-table :data="rows" row-key="id">
|
||||
<el-table-column prop="name" label="名称" min-width="180" />
|
||||
<el-table-column v-if="tab === 'flows' || tab === 'artifacts'" prop="revision" label="版本" width="90" />
|
||||
<el-table-column v-if="tab === 'tasks'" label="状态" width="120"><template #default="{ row }">{{ statuses[row.status] || row.status }}</template></el-table-column>
|
||||
<el-table-column v-if="tab === 'services'" label="状态" width="90"><template #default="{ row }">{{ row.enabled ? '启用' : '停用' }}</template></el-table-column>
|
||||
<el-table-column prop="updated_at" label="更新时间" min-width="190" />
|
||||
<el-table-column label="操作" min-width="300"><template #default="{ row }">
|
||||
<el-button link type="primary" @click="open(row)">{{ tab === 'flows' ? '查看 / 新版本' : tab === 'artifacts' ? '打开 / 复核' : '详情' }}</el-button>
|
||||
<template v-if="tab === 'services' && resources.is_admin">
|
||||
<el-button link type="primary" @click="toggle(row)">{{ row.enabled ? '停用' : '启用' }}</el-button>
|
||||
<el-button link type="primary" @click="rotate(row)">更新Key</el-button>
|
||||
<el-button v-if="row.key_configured" link @click="revoke(row)">撤销Key</el-button>
|
||||
</template>
|
||||
<el-button v-if="tab === 'tasks' && row.status === 'FAILED'" link type="primary" @click="retry(row)">重试</el-button>
|
||||
</template></el-table-column>
|
||||
</el-table>
|
||||
<el-dialog v-model="dialog" :title="title" width="740px" destroy-on-close>
|
||||
<el-form label-position="top">
|
||||
<el-form-item v-if="tab !== 'tasks'" label="名称"><el-input v-model="form.name" maxlength="128" /></el-form-item>
|
||||
<template v-if="tab === 'flows'">
|
||||
<el-form-item label="文档解析"><el-select v-model="form.parser_id" clearable placeholder="不解析文件"><el-option v-for="p in resources.parsers" :key="p.id" :label="p.name" :value="p.id"/></el-select></el-form-item>
|
||||
<el-form-item label="智能整理"><el-select v-model="form.model_id" clearable placeholder="按原文生成"><el-option v-for="m in llms" :key="m.id" :label="m.name" :value="m.id"/></el-select></el-form-item>
|
||||
<el-form-item v-if="form.model_id" label="提示词"><el-input v-model="form.prompt" type="textarea" :rows="4" /></el-form-item>
|
||||
<el-form-item v-if="form.model_id" label="温度"><el-input-number v-model="form.temperature" :min="0" :max="2" :step="0.1" /></el-form-item>
|
||||
<el-form-item label="输出">思维导图</el-form-item>
|
||||
</template>
|
||||
<template v-if="tab === 'services'">
|
||||
<el-form-item label="流程版本"><el-select v-model="form.flow_id" :disabled="!!form.id"><el-option v-for="f in flows" :key="f.id" :label="`${f.name} · v${f.revision}`" :value="f.id" /></el-select></el-form-item>
|
||||
<el-form-item v-if="!form.id" label="每分钟调用次数"><el-input-number v-model="form.requests_per_minute" :min="1" :max="600"/></el-form-item>
|
||||
<el-form-item v-if="form.id" label="服务接口"><el-input :model-value="serviceUrl(form)" readonly /></el-form-item>
|
||||
<el-form-item v-if="form.id" label="调用"><el-input v-model="runMarkdown" type="textarea" :rows="3" placeholder="Markdown"/><el-input v-model="runKey" type="password" show-password placeholder="服务Key(如已启用)"/><input type="file" @change="runFile = ($event.target as HTMLInputElement).files?.[0] || null"/><el-button @click="runService">提交任务</el-button></el-form-item>
|
||||
</template>
|
||||
<el-form-item v-if="resources.is_admin && tab === 'services'" label="授权用户">
|
||||
<el-select v-model="form.allowed_users" multiple filterable><el-option v-for="u in resources.users || []" :key="u.id" :label="u.name" :value="u.id"/></el-select>
|
||||
</el-form-item>
|
||||
<template v-if="tab === 'tasks'">
|
||||
<el-descriptions :column="1" border><el-descriptions-item label="任务ID">{{ form.id }}</el-descriptions-item><el-descriptions-item label="状态">{{ statuses[form.status] }}</el-descriptions-item><el-descriptions-item label="执行次数">{{ form.attempt }}</el-descriptions-item><el-descriptions-item v-if="form.error" label="失败原因">{{ form.error }}</el-descriptions-item></el-descriptions>
|
||||
<el-table :data="form.steps || []"><el-table-column label="节点"><template #default="{ row }">{{ nodeLabels[row.type] }}</template></el-table-column><el-table-column label="状态"><template #default="{ row }">{{ statuses[row.status] }}</template></el-table-column></el-table>
|
||||
<el-form-item label="解析原稿"><el-input :model-value="form.result?.original" type="textarea" :rows="5" readonly /></el-form-item>
|
||||
<el-form-item label="复核内容"><el-input v-model="reviewMarkdown" type="textarea" :rows="8" /></el-form-item>
|
||||
<el-button v-if="form.status === 'SUCCESS'" type="primary" @click="saveTaskResult">确认保存图谱</el-button>
|
||||
</template>
|
||||
<template v-if="tab === 'artifacts'">
|
||||
<el-form-item label="原始识别内容"><el-input :model-value="form.original" type="textarea" :rows="5" readonly/></el-form-item>
|
||||
<el-form-item label="修订内容"><el-input v-model="form.markdown" type="textarea" :rows="10"/></el-form-item>
|
||||
<el-form-item label="布局"><el-select v-model="form.layout"><el-option v-for="(name, value) in layouts" :key="value" :label="name" :value="value"/></el-select></el-form-item>
|
||||
<el-form-item label="历史修订"><el-select placeholder="查看历史" @change="showHistory"><el-option v-for="h in form.history || []" :key="h.revision" :label="`v${h.revision}`" :value="h.revision"/></el-select></el-form-item>
|
||||
<el-button @click="emit('open-artifact', form); dialog = false">在工作台打开</el-button>
|
||||
</template>
|
||||
</el-form>
|
||||
<template #footer><el-button @click="dialog = false">关闭</el-button><el-button v-if="canSave" type="primary" :loading="saving" @click="save">{{ tab === 'flows' && form.id ? '保存新版本' : '保存' }}</el-button></template>
|
||||
</el-dialog>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import { hubGet, hubPost, hubBase } from '@/api/ai-hub'
|
||||
import { request } from '@/request/index'
|
||||
const emit = defineEmits(['open-artifact'])
|
||||
const tab = ref('services'), loading = ref(false), saving = ref(false), dialog = ref(false)
|
||||
const rows = ref<any[]>([]), flows = ref<any[]>([]), form = ref<any>({}), resources = ref<any>({ parsers: [], models: [] })
|
||||
const reviewMarkdown = ref(''), runMarkdown = ref(''), runKey = ref(''), runFile = ref<File | null>(null)
|
||||
const llms = computed(() => resources.value.models.filter((m: any) => m.model_type === 'LLM'))
|
||||
const statuses: Record<string, string> = { PENDING: '待处理', RUNNING: '处理中', SUCCESS: '成功', FAILED: '失败' }
|
||||
const nodeLabels: Record<string, string> = { parse: '文档解析', organize: '智能整理', render: '导图生成' }
|
||||
const layouts = { tree: '树状' }
|
||||
const title = computed(() => ({ services: '导图服务', flows: '处理流程', tasks: '任务详情', artifacts: '图谱复核' })[tab.value as 'services'])
|
||||
const canSave = computed(() => ['flows', 'artifacts'].includes(tab.value) || (resources.value.is_admin && tab.value === 'services'))
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const [r, list, f] = await Promise.all([hubGet('resources'), hubGet(tab.value), hubGet('flows')])
|
||||
resources.value = r.data; rows.value = list.data; flows.value = f.data
|
||||
} finally { loading.value = false }
|
||||
}
|
||||
function create() { form.value = { name: '', allowed_users: [], enabled: true, parser_id: '', model_id: '', prompt: '保留原文事实、数值和语言,整理为分级Markdown标题和要点。', temperature: 0.2, requests_per_minute: 20 }; dialog.value = true }
|
||||
async function open(row: any) {
|
||||
form.value = (await hubGet(tab.value, row.id)).data
|
||||
if (tab.value === 'flows') {
|
||||
const p = form.value.nodes.find((n: any) => n.type === 'parse'), m = form.value.nodes.find((n: any) => n.type === 'organize')
|
||||
Object.assign(form.value, { parser_id: p?.resource_id || '', model_id: m?.model_id || '', prompt: m?.prompt || '', temperature: m?.temperature ?? 0.2 })
|
||||
}
|
||||
reviewMarkdown.value = form.value.result?.markdown || ''; runMarkdown.value = ''; runKey.value = ''; runFile.value = null
|
||||
dialog.value = true
|
||||
}
|
||||
async function save() {
|
||||
saving.value = true
|
||||
try {
|
||||
let body: any = { ...form.value }, id = form.value.id
|
||||
if (tab.value === 'flows') {
|
||||
body = { name: body.name, previous_id: id, nodes: [] }; id = undefined
|
||||
if (form.value.parser_id) body.nodes.push({ type: 'parse', resource_id: form.value.parser_id })
|
||||
if (form.value.model_id) body.nodes.push({ type: 'organize', model_id: form.value.model_id, prompt: form.value.prompt, temperature: form.value.temperature })
|
||||
body.nodes.push({ type: 'render' })
|
||||
}
|
||||
await hubPost(tab.value, body, id); dialog.value = false; ElMessage.success('已保存'); await load()
|
||||
} finally { saving.value = false }
|
||||
}
|
||||
async function toggle(row: any) { await hubPost('services', { enabled: !row.enabled }, row.id); await load() }
|
||||
async function retry(row: any) { await hubPost('tasks', {}, row.id); await load() }
|
||||
async function rotate(row: any) { await ElMessageBox.confirm('更新后原Key失效', '更新Key'); const r = await hubPost('services', { rotate_key: true }, row.id); await ElMessageBox.alert(r.data.api_key, '请保存Key'); await load() }
|
||||
async function revoke(row: any) { await ElMessageBox.confirm('撤销后使用登录权限访问', '撤销Key'); await hubPost('services', { revoke_key: true }, row.id); await load() }
|
||||
function serviceUrl(row: any) { return `${location.origin}${import.meta.env.BASE_URL}api${hubBase()}/published/${row.id}/run` }
|
||||
async function runService() {
|
||||
const data = new FormData(); data.append('data', JSON.stringify({ markdown: runMarkdown.value }))
|
||||
if (runFile.value) data.append('file', runFile.value)
|
||||
await request({ url: `${hubBase()}/published/${form.value.id}/run`, method: 'post', data, headers: { 'X-Service-Key': runKey.value }, timeout: 30000 })
|
||||
ElMessage.success('任务已提交'); dialog.value = false; tab.value = 'tasks'; await load()
|
||||
}
|
||||
async function saveTaskResult() { await hubPost('artifacts', { name: form.value.name, task_id: form.value.id, markdown: reviewMarkdown.value, layout: 'tree' }); ElMessage.success('已保存'); dialog.value = false; tab.value = 'artifacts'; await load() }
|
||||
function showHistory(revision: number) { const h = form.value.history.find((item: any) => item.revision === revision); if (h) ElMessageBox.alert(h.markdown, `v${revision}`, { customStyle: { maxWidth: '80vw', whiteSpace: 'pre-wrap', maxHeight: '80vh', overflow: 'auto' } }) }
|
||||
onMounted(load)
|
||||
</script>
|
||||
<style scoped>.hub-manager { padding: 16px 24px; height: 100%; overflow: auto; } .el-select { width: 100%; }</style>
|
||||
|
|
@ -20,13 +20,13 @@ import { ref, onMounted, onBeforeUnmount } from 'vue'
|
|||
import { useRoute } from 'vue-router'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import systemConfigApi from '@/api/system-settings/system-config'
|
||||
import { hubGet, hubPost } from '@/api/ai-hub'
|
||||
import { hubGet, hubPost, hubBinary } from '@/api/ai-hub'
|
||||
|
||||
const mindmapUrl = ref<string>('')
|
||||
const loading = ref(false)
|
||||
const route = useRoute()
|
||||
const frame = ref<HTMLIFrameElement>()
|
||||
const allowedKinds = new Set(['resources', 'tasks', 'artifacts'])
|
||||
const allowedKinds = new Set(['resources', 'tasks', 'artifacts', 'assets', 'exports'])
|
||||
function childOrigin() { try { return new URL(mindmapUrl.value, location.href).origin } catch { return '' } }
|
||||
async function openRequestedArtifact() {
|
||||
const id = route.query.artifact
|
||||
|
|
@ -43,6 +43,12 @@ async function handleBridge(event: MessageEvent) {
|
|||
const { id, kind, recordId, method, data, file } = event.data
|
||||
if (typeof id !== 'string' || id.length > 128 || !allowedKinds.has(kind) || !['get', 'post'].includes(method) || (recordId && !/^[a-f0-9-]{36}$/i.test(recordId))) return
|
||||
try {
|
||||
if (kind === 'assets' || kind === 'exports') {
|
||||
if ((kind === 'assets' && (method !== 'get' || !recordId)) || (kind === 'exports' && (method !== 'post' || recordId))) throw new Error('请求无效')
|
||||
const binary = await hubBinary(kind, recordId, data)
|
||||
frame.value?.contentWindow?.postMessage({ type: 'ai-hub-response', id, data: binary }, event.origin, [binary.content])
|
||||
return
|
||||
}
|
||||
let payload = data
|
||||
if (file) {
|
||||
if (!(file.content instanceof ArrayBuffer) || file.content.byteLength > 20 * 1024 * 1024 || typeof file.name !== 'string') throw new Error('文件无效')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
<template>
|
||||
<div v-loading="loading">
|
||||
<el-radio-group v-model="mode" class="mb-16">
|
||||
<el-radio-button value="PATH">服务器目录</el-radio-button>
|
||||
<el-radio-button value="UPLOAD">上传ZIP</el-radio-button>
|
||||
</el-radio-group>
|
||||
<template v-if="mode === 'PATH'">
|
||||
<el-input v-model="path" placeholder="服务器模型绝对目录" class="mb-12" />
|
||||
</template>
|
||||
<el-upload v-else :auto-upload="false" :limit="1" accept=".zip" v-model:file-list="files" class="mb-12" :on-exceed="() => MsgError('请先移除已选文件')">
|
||||
<el-button>选择模型包</el-button>
|
||||
<template #tip><div class="el-upload__tip">ZIP ≤ 2GB;大模型使用服务器目录</div></template>
|
||||
</el-upload>
|
||||
<el-button type="primary" @click="importFiles" :disabled="mode === 'PATH' ? !path.trim() : !files.length">导入</el-button>
|
||||
<div class="color-secondary mt-12 mb-12">文件导入不启动推理服务</div>
|
||||
<el-table :data="records" max-height="300">
|
||||
<el-table-column prop="name" label="名称" min-width="140" show-overflow-tooltip />
|
||||
<el-table-column prop="format" label="格式" width="80" />
|
||||
<el-table-column label="大小" width="85"><template #default="{ row }">{{ (row.size_bytes / 1024 ** 2).toFixed(1) }} MB</template></el-table-column>
|
||||
<el-table-column label="操作" width="110"><template #default="{ row }">
|
||||
<el-button link type="primary" @click="selected = row">详情</el-button>
|
||||
<el-button v-if="row.format === 'HF'" link type="primary" @click="$emit('use', row)">使用</el-button>
|
||||
</template></el-table-column>
|
||||
</el-table>
|
||||
<el-descriptions v-if="selected" :column="1" :label-width="80" border size="small" class="mt-16 import-details">
|
||||
<el-descriptions-item label="目录">{{ selected.path }}</el-descriptions-item>
|
||||
<el-descriptions-item label="文件">{{ selected.file_count }} 个,权重 {{ selected.weight_count }} 个</el-descriptions-item>
|
||||
<el-descriptions-item label="元信息">{{ JSON.stringify(selected.metadata) }}</el-descriptions-item>
|
||||
<el-descriptions-item label="文件检查">{{ selected.checks.join(';') }}</el-descriptions-item>
|
||||
<el-descriptions-item label="运行状态">未验证</el-descriptions-item>
|
||||
</el-descriptions>
|
||||
</div>
|
||||
</template>
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref } from 'vue'
|
||||
import type { UploadUserFile } from 'element-plus'
|
||||
import { get, post } from '@/request'
|
||||
import { MsgError, MsgSuccess } from '@/utils/message'
|
||||
defineEmits(['use'])
|
||||
const mode = ref('PATH'), path = ref(''), files = ref<UploadUserFile[]>([])
|
||||
const loading = ref(false), records = ref<any[]>([]), selected = ref<any>()
|
||||
async function load() { records.value = (await get('/local_model/imports', undefined, loading)).data }
|
||||
async function importFiles() {
|
||||
let body: any = { path: path.value.trim() }
|
||||
if (mode.value === 'UPLOAD') {
|
||||
const file = files.value[0]?.raw
|
||||
if (!file || !file.name.toLowerCase().endsWith('.zip') || file.size > 2 * 1024 ** 3) { MsgError('请选择2GB以内的ZIP模型包'); return }
|
||||
body = new FormData(); body.append('file', file)
|
||||
}
|
||||
try {
|
||||
selected.value = (await post('/local_model/imports', body, undefined, loading)).data
|
||||
MsgSuccess('已导入'); path.value = ''; files.value = []; await load()
|
||||
} catch { /* request interceptor displays the server error; retain the form */ }
|
||||
}
|
||||
onMounted(load)
|
||||
</script>
|
||||
<style scoped>
|
||||
.import-details :deep(.el-descriptions__content) { overflow-wrap: anywhere; word-break: break-word; }
|
||||
</style>
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
<template>
|
||||
<el-dialog v-model="visible" title="模型载入" width="560px" :close-on-click-modal="false" @closed="stopPolling">
|
||||
<el-form label-position="top" v-loading="saving">
|
||||
<el-form-item label="模型"><el-input :model-value="model.name" readonly /></el-form-item>
|
||||
<el-form-item label="加载方式">
|
||||
<el-select v-model="form.strategy" class="w-full">
|
||||
<el-option label="按需加载" value="ON_DEMAND" />
|
||||
<el-option label="启动预加载" value="PRELOAD" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="加载优先级(数值越大越优先)">
|
||||
<el-input-number v-model="form.priority" :min="0" :max="100" :precision="0" controls-position="right" />
|
||||
</el-form-item>
|
||||
<div class="flex-between mb-12">
|
||||
<span>运行状态 <el-tag :type="state.status === 'READY' ? 'success' : state.status === 'FAILED' ? 'danger' : 'info'" class="ml-8">{{ statusLabel }}</el-tag></span>
|
||||
<el-button link type="primary" :loading="checking" @click="refresh">刷新</el-button>
|
||||
</div>
|
||||
<div v-if="state.device" class="color-secondary mb-8">
|
||||
设备:{{ state.device }}
|
||||
<span v-if="state.gpu_bytes !== undefined"> · 显存:{{ (state.gpu_bytes / 1024 / 1024).toFixed(0) }} MB</span>
|
||||
<span v-if="state.process_gpu_bytes !== undefined"> · 服务进程显存:{{ (state.process_gpu_bytes / 1024 / 1024).toFixed(0) }} MB</span>
|
||||
</div>
|
||||
<div v-if="state.loaded_at" class="color-secondary mb-8">加载时间:{{ new Date(state.loaded_at * 1000).toLocaleString() }}</div>
|
||||
<div v-if="state.active_requests" class="color-secondary mb-8">当前请求:{{ state.active_requests }}</div>
|
||||
<div v-if="state.error" class="color-danger mb-12">{{ state.error }}</div>
|
||||
<el-button :disabled="dirty || busy || state.status === 'READY' || !ready" :loading="acting === 'load'" @click="run('load')">立即加载</el-button>
|
||||
<el-button :disabled="dirty || busy || state.status !== 'READY' || !!state.active_requests" :loading="acting === 'unload'" @click="run('unload')">卸载</el-button>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="visible = false">关闭</el-button>
|
||||
<el-button type="primary" :loading="saving" :disabled="!ready || !!acting" @click="save">保存</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, reactive, computed, onBeforeUnmount } from 'vue'
|
||||
import { get, put, post } from '@/request'
|
||||
import useStore from '@/stores'
|
||||
import { MsgSuccess, MsgError } from '@/utils/message'
|
||||
|
||||
const { user } = useStore()
|
||||
const emit = defineEmits(['change'])
|
||||
const visible = ref(false), ready = ref(false), checking = ref(false), saving = ref(false)
|
||||
const acting = ref(''), model = ref<any>({}), state = ref<any>({})
|
||||
const form = reactive({ strategy: 'ON_DEMAND', priority: 50, idle_seconds: 300 })
|
||||
const saved = ref('')
|
||||
const dirty = computed(() => JSON.stringify(form) !== saved.value)
|
||||
const busy = computed(() => !!acting.value || saving.value || ['LOADING', 'QUEUED'].includes(state.value.status))
|
||||
const labels: Record<string, string> = { UNLOADED: '未加载', QUEUED: '等待加载', LOADING: '加载中', READY: '已加载', FAILED: '加载失败', UNAVAILABLE: '服务不可用', EXTERNAL: '外部服务' }
|
||||
const statusLabel = computed(() => labels[state.value.status] || '读取中')
|
||||
let timer: ReturnType<typeof setInterval> | undefined
|
||||
const path = () => `/workspace/${model.value.workspace_id || user.getWorkspaceId()}/model/${model.value.id}/runtime`
|
||||
|
||||
function stopPolling() { if (timer) clearInterval(timer); timer = undefined }
|
||||
async function refresh() {
|
||||
if (checking.value || !visible.value) return
|
||||
const id = model.value.id
|
||||
const response: any = await get(path(), undefined, checking)
|
||||
if (visible.value && model.value.id === id) state.value = response.data
|
||||
}
|
||||
async function open(value: any) {
|
||||
stopPolling(); model.value = value; state.value = {}; ready.value = false; visible.value = true
|
||||
const id = value.id
|
||||
const response: any = await get(path(), undefined, checking)
|
||||
if (!visible.value || model.value.id !== id) return
|
||||
state.value = response.data
|
||||
Object.assign(form, { strategy: response.data.strategy, priority: response.data.priority, idle_seconds: response.data.idle_seconds })
|
||||
saved.value = JSON.stringify(form); ready.value = response.data.supported
|
||||
timer = setInterval(() => { refresh().catch(() => {}) }, 5000)
|
||||
}
|
||||
async function save() {
|
||||
if (!Number.isInteger(form.priority) || form.priority < 0 || form.priority > 100) { MsgError('加载优先级须为0~100的整数'); return }
|
||||
const response: any = await put(path(), { ...form }, undefined, saving)
|
||||
state.value = response.data; saved.value = JSON.stringify(form)
|
||||
emit('change'); MsgSuccess('保存成功')
|
||||
}
|
||||
async function run(action: string) {
|
||||
if (busy.value || dirty.value) return
|
||||
acting.value = action
|
||||
try {
|
||||
const response: any = await post(path(), { action })
|
||||
state.value = response.data
|
||||
MsgSuccess(action === 'load' ? '加载请求已提交' : '已卸载')
|
||||
} finally { acting.value = '' }
|
||||
}
|
||||
onBeforeUnmount(stopPolling)
|
||||
defineExpose({ open })
|
||||
</script>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
<template>
|
||||
<div class="mindmap-management p-16-24">
|
||||
<h2 class="mb-16">思维导图管理</h2>
|
||||
<el-card :body-style="{ padding: '0' }">
|
||||
<HubManager @open-artifact="openArtifact" />
|
||||
</el-card>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { useRouter } from 'vue-router'
|
||||
import HubManager from '@/views/mindmap/HubManager.vue'
|
||||
|
||||
const router = useRouter()
|
||||
function openArtifact(artifact: { id: string }) {
|
||||
router.push({ path: '/mindmap', query: { artifact: artifact.id } })
|
||||
}
|
||||
</script>
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
<template>
|
||||
<el-drawer v-model="visible" size="min(1180px, 100vw)" title="待激活审核" :destroy-on-close="false">
|
||||
<div class="audit-toolbar">
|
||||
<el-input
|
||||
v-model="keyword"
|
||||
placeholder="搜索用户名 / 姓名 / 邮箱 / 手机号"
|
||||
clearable
|
||||
style="width: 280px"
|
||||
@keyup.enter="search"
|
||||
/>
|
||||
<el-select v-model="status" style="width: 120px" @change="search">
|
||||
<el-option label="全部" value="UNAPPROVED" />
|
||||
<el-option label="待审核" value="PENDING" />
|
||||
<el-option label="已拒绝" value="REJECTED" />
|
||||
</el-select>
|
||||
<el-button type="primary" @click="search">查询</el-button>
|
||||
<el-button @click="getList">刷新</el-button>
|
||||
<span class="audit-count">共 <b>{{ total }}</b> 人</span>
|
||||
</div>
|
||||
|
||||
<el-table :data="records" v-loading="loading" height="calc(100vh - 235px)">
|
||||
<el-table-column prop="nick_name" label="昵称" width="90" show-overflow-tooltip />
|
||||
<el-table-column prop="username" label="用户名" width="110" show-overflow-tooltip />
|
||||
<el-table-column prop="email" label="注册邮箱" min-width="180" show-overflow-tooltip />
|
||||
<el-table-column prop="phone" label="手机号" width="130">
|
||||
<template #default="{ row }">{{ row.phone || '-' }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="邮箱验证" width="90" align="center">
|
||||
<template #default="{ row }">
|
||||
<el-tag v-if="row.email_verified" type="success" size="small">已验证</el-tag>
|
||||
<el-tag v-else type="info" size="small">未验证</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="applied_at" label="申请时间" width="160" />
|
||||
<el-table-column label="状态" width="90">
|
||||
<template #default="{ row }">
|
||||
<el-tag v-if="row.status === 'REJECTED'" type="danger" size="small">已拒绝</el-tag>
|
||||
<el-tag v-else type="warning" size="small">待审核</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="reject_reason" label="拒绝原因" min-width="150" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ row.reject_reason || '-' }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="操作" width="120" fixed="right">
|
||||
<template #default="{ row }">
|
||||
<template v-if="row.status === 'PENDING'">
|
||||
<el-button type="primary" text @click="approve(row)">通过</el-button>
|
||||
<el-button type="danger" text @click="reject(row)">拒绝</el-button>
|
||||
</template>
|
||||
<span v-else>-</span>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<template #empty>
|
||||
<el-empty description="暂无记录" />
|
||||
</template>
|
||||
</el-table>
|
||||
|
||||
<div class="audit-pagination">
|
||||
<el-pagination
|
||||
v-model:current-page="pagination.current_page"
|
||||
v-model:page-size="pagination.page_size"
|
||||
:page-sizes="[10, 20, 50]"
|
||||
:total="total"
|
||||
layout="total, prev, pager, next, sizes"
|
||||
@size-change="getList"
|
||||
@current-change="getList"
|
||||
/>
|
||||
</div>
|
||||
</el-drawer>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { reactive, ref } from 'vue'
|
||||
import { ElMessageBox } from 'element-plus'
|
||||
import userManageApi from '@/api/system/user-manage'
|
||||
import { MsgSuccess } from '@/utils/message'
|
||||
import { t } from '@/locales'
|
||||
|
||||
const emit = defineEmits(['refresh'])
|
||||
const visible = ref(false)
|
||||
const loading = ref(false)
|
||||
const keyword = ref('')
|
||||
const status = ref('UNAPPROVED')
|
||||
const records = ref<any[]>([])
|
||||
const total = ref(0)
|
||||
const pagination = reactive({ current_page: 1, page_size: 10 })
|
||||
|
||||
function open() {
|
||||
visible.value = true
|
||||
pagination.current_page = 1
|
||||
keyword.value = ''
|
||||
status.value = 'UNAPPROVED'
|
||||
getList()
|
||||
}
|
||||
|
||||
function search() {
|
||||
pagination.current_page = 1
|
||||
getList()
|
||||
}
|
||||
|
||||
function getList() {
|
||||
loading.value = true
|
||||
userManageApi
|
||||
.getActivationPending(
|
||||
{
|
||||
current_page: pagination.current_page,
|
||||
page_size: pagination.page_size,
|
||||
keyword: keyword.value,
|
||||
status: status.value,
|
||||
},
|
||||
loading,
|
||||
)
|
||||
.then((res: any) => {
|
||||
records.value = res.data?.records || []
|
||||
total.value = res.data?.total || 0
|
||||
})
|
||||
.finally(() => {
|
||||
loading.value = false
|
||||
})
|
||||
}
|
||||
|
||||
function approve(row: any) {
|
||||
ElMessageBox.confirm(
|
||||
`确认激活「${row.nick_name || row.username}」?`,
|
||||
'审核通过',
|
||||
{
|
||||
confirmButtonText: '确认通过',
|
||||
cancelButtonText: t('common.cancel'),
|
||||
type: 'warning',
|
||||
},
|
||||
)
|
||||
.then(() => {
|
||||
userManageApi
|
||||
.approveActivation(row.id, { role_id: 'USER', workspace_id: 'default' }, loading)
|
||||
.then(() => {
|
||||
MsgSuccess('已激活')
|
||||
getList()
|
||||
emit('refresh')
|
||||
})
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
|
||||
function reject(row: any) {
|
||||
ElMessageBox.prompt('拒绝原因', '审核拒绝', {
|
||||
confirmButtonText: '确认拒绝',
|
||||
cancelButtonText: t('common.cancel'),
|
||||
inputPlaceholder: '请输入拒绝原因',
|
||||
inputValidator: (value: string) => (value && value.trim() ? true : '请填写拒绝原因'),
|
||||
})
|
||||
.then(({ value }: any) => {
|
||||
userManageApi
|
||||
.rejectActivation(row.id, { reason: value }, loading)
|
||||
.then(() => {
|
||||
MsgSuccess('已拒绝')
|
||||
getList()
|
||||
emit('refresh')
|
||||
})
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
|
||||
defineExpose({ open })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.audit-toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
.audit-count {
|
||||
margin-left: auto;
|
||||
color: #646a73;
|
||||
font-size: 13px;
|
||||
}
|
||||
.audit-count b {
|
||||
color: #3370ff;
|
||||
font-size: 15px;
|
||||
margin: 0 2px;
|
||||
}
|
||||
.audit-pagination {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
margin-top: 12px;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
<template>
|
||||
<el-dialog v-model="visible" title="分配角色" width="580px" :close-on-click-modal="false">
|
||||
<div class="role-user">
|
||||
<div class="role-user__avatar">{{ (detail.user?.nick_name || detail.user?.username || '?').slice(0, 1) }}</div>
|
||||
<div class="role-user__info">
|
||||
<div class="role-user__name">
|
||||
{{ detail.user?.nick_name || '-' }}
|
||||
<el-tag v-if="detail.user?.is_active" type="success" size="small">已启用</el-tag>
|
||||
<el-tag v-else type="info" size="small">未启用</el-tag>
|
||||
</div>
|
||||
<div class="role-user__meta">{{ detail.user?.username }} · {{ detail.user?.email || '未填写邮箱' }}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<el-form label-position="top" class="role-form">
|
||||
<el-form-item label="角色">
|
||||
<el-select v-model="form.role_id" placeholder="请选择角色" style="width: 100%">
|
||||
<el-option
|
||||
v-for="item in availableRoles"
|
||||
:key="item.id"
|
||||
:label="item.name"
|
||||
:value="item.id"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
|
||||
<div class="role-current">
|
||||
<div class="role-current__title">当前授权</div>
|
||||
<el-table :data="detail.assignments" size="small" border>
|
||||
<el-table-column label="角色" min-width="120">
|
||||
<template #default="{ row }">{{ roleName(row.role_id) }}</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="granted_at" label="授权时间" min-width="160">
|
||||
<template #default="{ row }">{{ row.granted_at || '-' }}</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</div>
|
||||
|
||||
<template #footer>
|
||||
<el-button @click="visible = false">{{ $t('common.cancel') }}</el-button>
|
||||
<el-button type="primary" :loading="loading" :disabled="!form.role_id" @click="save">保存</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { reactive, ref } from 'vue'
|
||||
import userManageApi from '@/api/system/user-manage'
|
||||
import { MsgSuccess } from '@/utils/message'
|
||||
|
||||
const emit = defineEmits(['refresh'])
|
||||
const visible = ref(false)
|
||||
const loading = ref(false)
|
||||
const detail = reactive<any>({ user: {}, assignments: [] })
|
||||
const form = reactive({ role_id: '' })
|
||||
const availableRoles = [
|
||||
{ id: 'USER', name: '普通用户' },
|
||||
{ id: 'ADMIN', name: '管理员' },
|
||||
]
|
||||
|
||||
const roleMap: Record<string, string> = {
|
||||
ADMIN: '管理员',
|
||||
USER: '普通用户',
|
||||
}
|
||||
|
||||
function roleName(id: string) {
|
||||
return roleMap[id] || id
|
||||
}
|
||||
|
||||
function open(row: any) {
|
||||
visible.value = true
|
||||
loading.value = true
|
||||
detail.user = {}
|
||||
detail.assignments = []
|
||||
form.role_id = ''
|
||||
userManageApi
|
||||
.getUserRoles(row.id, loading)
|
||||
.then((res: any) => {
|
||||
const data = res.data || {}
|
||||
detail.user = data.user || row
|
||||
detail.assignments = (data.assignments || []).filter((item: any) => item.role_id in roleMap)
|
||||
const first = detail.assignments?.[0]
|
||||
const currentRole = data.user?.role || first?.role_id
|
||||
form.role_id = currentRole in roleMap ? currentRole : ''
|
||||
})
|
||||
.finally(() => {
|
||||
loading.value = false
|
||||
})
|
||||
}
|
||||
|
||||
function save() {
|
||||
if (!detail.user?.id || !form.role_id || loading.value) return
|
||||
loading.value = true
|
||||
userManageApi
|
||||
.putUserRoles(
|
||||
detail.user.id,
|
||||
{
|
||||
role_settings: [
|
||||
{
|
||||
role_id: form.role_id,
|
||||
},
|
||||
],
|
||||
},
|
||||
loading,
|
||||
)
|
||||
.then(() => {
|
||||
MsgSuccess('角色已更新')
|
||||
visible.value = false
|
||||
emit('refresh')
|
||||
})
|
||||
.finally(() => {
|
||||
loading.value = false
|
||||
})
|
||||
}
|
||||
|
||||
defineExpose({ open })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.role-user {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 4px 0 12px;
|
||||
}
|
||||
.role-user__avatar {
|
||||
width: 42px;
|
||||
height: 42px;
|
||||
border-radius: 8px;
|
||||
background: #e6f0ff;
|
||||
color: #3370ff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 18px;
|
||||
font-weight: 700;
|
||||
}
|
||||
.role-user__name {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: #1f2329;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.role-user__meta {
|
||||
color: #646a73;
|
||||
font-size: 12.5px;
|
||||
margin-top: 3px;
|
||||
}
|
||||
.role-form {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.role-current__title {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #1f2329;
|
||||
margin: 10px 0 8px;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
<template>
|
||||
<div class="parser-manager" v-loading="loading">
|
||||
<div class="flex-between mb-16">
|
||||
<h4>文档解析</h4>
|
||||
<div><el-button @click="load">刷新</el-button><el-button v-if="resources.is_admin" type="primary" @click="create">新增解析服务</el-button></div>
|
||||
</div>
|
||||
<el-table :data="rows" row-key="id">
|
||||
<el-table-column prop="name" label="名称" min-width="180" />
|
||||
<el-table-column label="接口类型" width="160"><template #default="{ row }">{{ row.options?.protocol === 'v1' ? 'MinerU v1' : 'MinerU 原版' }}</template></el-table-column>
|
||||
<el-table-column label="状态" width="100"><template #default="{ row }">{{ row.enabled ? '启用' : '停用' }}</template></el-table-column>
|
||||
<el-table-column prop="updated_at" label="更新时间" min-width="190"><template #default="{ row }">{{ new Date(row.updated_at).toLocaleString() }}</template></el-table-column>
|
||||
<el-table-column v-if="resources.is_admin" label="操作" min-width="230"><template #default="{ row }">
|
||||
<el-button link type="primary" @click="open(row)">编辑</el-button>
|
||||
<el-button link type="primary" :loading="testing === row.id" :disabled="!!testing && testing !== row.id" @click="test(row)">连接测试</el-button>
|
||||
<el-button link type="primary" @click="toggle(row)">{{ row.enabled ? '停用' : '启用' }}</el-button>
|
||||
</template></el-table-column>
|
||||
<template #empty>暂无解析服务</template>
|
||||
</el-table>
|
||||
<el-dialog v-model="dialog" :title="form.id ? '编辑解析服务' : '新增解析服务'" width="640px" destroy-on-close :close-on-click-modal="false">
|
||||
<el-form ref="formRef" :model="form" label-position="top">
|
||||
<el-form-item label="名称" prop="name" :rules="[{ required: true, message: '请输入名称', whitespace: true }]"><el-input v-model="form.name" maxlength="128" /></el-form-item>
|
||||
<el-form-item label="接口地址" prop="endpoint" :rules="[{ required: true, message: '请输入接口地址', whitespace: true }]"><el-input v-model="form.endpoint" placeholder="http://服务器:8100/v1" /></el-form-item>
|
||||
<el-form-item label="API Key"><el-input v-model="form.api_key" type="password" show-password :placeholder="form.credential_configured ? '留空保留原密钥' : ''" autocomplete="new-password" /></el-form-item>
|
||||
<el-form-item label="接口类型"><el-select v-model="form.options.protocol"><el-option label="MinerU v1 API" value="v1" /><el-option label="MinerU 原版API" value="legacy" /></el-select></el-form-item>
|
||||
<el-form-item v-if="form.options.protocol === 'v1'" label="解析档位"><el-select v-model="form.options.tier"><el-option v-for="tier in ['flash', 'basic', 'standard', 'advanced']" :key="tier" :label="tier" :value="tier" /></el-select></el-form-item>
|
||||
<template v-else>
|
||||
<el-form-item label="解析后端"><el-select v-model="form.options.backend"><el-option label="传统管道" value="pipeline" /><el-option label="混合引擎" value="hybrid-auto-engine" /><el-option label="视觉引擎" value="vlm-auto-engine" /></el-select></el-form-item>
|
||||
<el-form-item label="识别语言"><el-select v-model="form.options.language"><el-option v-for="item in languages" :key="item.value" v-bind="item" /></el-select></el-form-item>
|
||||
</template>
|
||||
<div class="mb-16"><el-checkbox v-model="form.options.ocr">强制OCR</el-checkbox><template v-if="form.options.protocol !== 'v1'"><el-checkbox v-model="form.options.table">表格</el-checkbox><el-checkbox v-model="form.options.formula">公式</el-checkbox></template></div>
|
||||
<el-form-item label="授权用户"><el-select v-model="form.allowed_users" multiple filterable><el-option v-for="u in resources.users || []" :key="u.id" :label="u.name" :value="u.id" /></el-select></el-form-item>
|
||||
<el-form-item label="启用"><el-switch v-model="form.enabled" /></el-form-item>
|
||||
</el-form>
|
||||
<template #footer><el-button @click="dialog = false">取消</el-button><el-button type="primary" :loading="saving" @click="save">保存</el-button></template>
|
||||
</el-dialog>
|
||||
</div>
|
||||
</template>
|
||||
<script setup lang="ts">
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import type { FormInstance } from 'element-plus'
|
||||
import { hubGet, hubPost, hubBase } from '@/api/ai-hub'
|
||||
import { post } from '@/request'
|
||||
const loading = ref(false), saving = ref(false), dialog = ref(false), testing = ref('')
|
||||
const resources = ref<any>({ is_admin: false, users: [] }), rows = ref<any[]>([]), form = ref<any>({ options: {} })
|
||||
const formRef = ref<FormInstance>()
|
||||
const languages = [{ value: 'ch', label: '中文' }, { value: 'en', label: '英文' }, { value: 'japan', label: '日文' }, { value: 'korean', label: '韩文' }, { value: 'chinese_cht', label: '繁体中文' }]
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try { const [r, p] = await Promise.all([hubGet('resources'), hubGet('parsers')]); resources.value = r.data; rows.value = p.data }
|
||||
finally { loading.value = false }
|
||||
}
|
||||
function create() { form.value = { name: '', endpoint: '', api_key: '', enabled: true, allowed_users: [], options: { protocol: 'v1', tier: 'basic', backend: 'pipeline', language: 'ch', table: true, formula: true, ocr: false } }; dialog.value = true }
|
||||
async function open(row: any) { form.value = (await hubGet('parsers', row.id)).data; form.value.api_key = ''; dialog.value = true }
|
||||
async function save() {
|
||||
if (!await formRef.value?.validate().catch(() => false)) return
|
||||
saving.value = true
|
||||
try { const body = { ...form.value }; if (!body.api_key) delete body.api_key; await hubPost('parsers', body, body.id); dialog.value = false; ElMessage.success('已保存'); await load() }
|
||||
finally { saving.value = false }
|
||||
}
|
||||
async function toggle(row: any) { loading.value = true; try { await hubPost('parsers', { enabled: !row.enabled }, row.id); await load() } finally { loading.value = false } }
|
||||
async function test(row: any) {
|
||||
testing.value = row.id
|
||||
try { const r = await post(`${hubBase()}/parsers/${row.id}/test`, {}); ElMessage.success(`连接正常(${r.data.elapsed_ms} ms)`) }
|
||||
finally { testing.value = '' }
|
||||
}
|
||||
onMounted(load)
|
||||
</script>
|
||||
<style scoped>.parser-manager { padding: 16px 24px; flex: 1; min-height: 0; overflow: auto; } .el-select { width: 100%; }</style>
|
||||
Loading…
Reference in New Issue