Compare commits

..

14 Commits

Author SHA1 Message Date
panyy d8e640748e feat(deploy): package MinerU 2.7.6 releases for CUDA and CoreX
build-release.sh built the parser through markitdown's own script and
shipped one images.tar, so every customer downloaded both accelerators'
layers. Build from Dockerfile.mineru.cuda.slim or Dockerfile.mineru.corex
instead and split KB, gateway and parser into separate archives. Add the
compat rebuild path, the customer fresh-install entry, and the 2.7.6
upgrade guide; README_UPGRADE.md only describes 2.7.1 and is now marked
historical.

upgrade.py needs the nvidia runtime set explicitly, probes /health rather
than /openapi.json, and asserts 2.7.6. Migration 0008 creates the unified
role tables but does not populate them, so run unify_platform_roles after
migrating.

Ignore ui/.codex-validation* in Git and in the KB build context: the
release script refuses a worktree with untracked files, and those 116M of
verification output were also entering the docker context.
2026-11-12 17:54:49 +08:00
panyy b23569e0b8 fix(mindmap): surface backend validation messages in the workbench
find_err_detail relied on ReturnDict.serializer, so plain string and list
errors lost their message and nested ones could raise. Walk dicts and
lists directly instead, and keep the internal non_field_errors key out of
user-facing text.

The axios interceptor toasted every failure before the page could handle
it, so saving over the mindmap limit showed two messages and the workbench
fell back to a generic one. Add an opt-in silentError flag and let the
workbench render the server message itself.
2026-11-12 17:54:06 +08:00
panyy 79fa2cf4a2 feat:思维导图2.7.6 2026-11-09 11:03:23 +08:00
panyy 2600fbf3a4 fix(ui): simplify document parser version and builtin labels 2026-11-06 17:08:25 +08:00
panyy 851cb6b484 fix(parser): align platform and upgrade checks with MinerU 2.7.1 2026-11-06 16:52:29 +08:00
panyy 4aea21b6a1 fix(deploy): verify bundled PostgreSQL and Redis after upgrade 2026-11-06 15:27:47 +08:00
panyy 5ee44927b5 feat(deploy): add isolated refact upgrades and CUDA CoreX release packaging 2026-11-06 15:00:00 +08:00
panyy 91f9199020 chore(release): include refact runtime sources and frontend lockfile 2026-11-06 14:51:58 +08:00
panyy 96a81496be feat:优化使用体验1 2026-11-06 14:05:34 +08:00
panyy 5434504f9b feat:优化使用体验 2026-11-05 18:16:48 +08:00
panyy 2baba0b19c feat:知识库版本 2026-10-27 13:37:24 +08:00
panyy e06b78918b feat: integrate AI workbench, unified permissions and access control 2026-10-27 10:54:52 +08:00
panyy 69eb1eb681 完善知识库、模型管理和思维导图功能 2026-10-22 14:48:00 +08:00
panyy 6df950ec3f feat:优化重构 2026-10-21 15:48:54 +08:00
333 changed files with 30177 additions and 2223 deletions

7
.gitignore vendored
View File

@ -190,3 +190,10 @@ apps/models_provider/impl/tencent_model_provider/model/stt.py
tmp/
config.yml
.SANDBOX_BANNED_HOSTS
# Local verification outputs; release sources remain visible to Git.
/ui/dist*/
/ui/.summary-build/
/ui/.codex-validation*/
/jieba.cache
/mcp-engine-*/

View File

@ -183,6 +183,10 @@ class BaseChatStep(IChatStep):
tool_ids=None,
mcp_output_enable=True,
**kwargs):
from system_manage.mcp_agent import context_for
enabled = (model_setting or {}).get('engine_tools', [])
self.context['engine_body'] = {**manage.context, 'workspace_id': workspace_id, 'chat_id': str(chat_id)}
self.context['engine_tools'] = enabled
chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id,
**model_params_setting) if model_id is not None else None
if stream:
@ -208,7 +212,9 @@ class BaseChatStep(IChatStep):
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
if os.path.exists(code_path):
os.remove(code_path)
from system_manage.mcp_agent import call_records
return {
'engine_calls': call_records(self.context.get('engine_body', {})),
'step_type': 'chat_step',
'run_time': self.context['run_time'],
'model_id': str(manage.context['model_id']),
@ -231,25 +237,15 @@ class BaseChatStep(IChatStep):
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_ids, tool_ids,
mcp_output_enable, chat_model, message_list):
if not mcp_enable and not tool_enable:
if not mcp_enable and not tool_enable and not self.context.get('engine_tools'):
return None
mcp_servers_config = {}
# 迁移过来mcp_source是None
if mcp_source is None:
mcp_source = 'custom'
if mcp_enable:
# 兼容老数据
if not mcp_tool_ids:
mcp_tool_ids = []
if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers:
mcp_servers_config = json.loads(mcp_servers)
elif mcp_tool_ids:
mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values()
for mcp_tool in mcp_tools:
if mcp_tool and mcp_tool['is_active']:
mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])}
from system_manage.mcp_agent import context_for
engine_context = context_for(self.context['engine_body'], engine_tools=self.context.get('engine_tools'),
tool_ids=mcp_tool_ids if mcp_enable and mcp_source not in (None, 'custom') else [],
custom=mcp_servers if mcp_enable and mcp_source in (None, 'custom') else None)
if tool_enable:
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
@ -269,8 +265,8 @@ class BaseChatStep(IChatStep):
self.context['execute_ids'].append(_id)
mcp_servers_config[str(tool.id)] = tool_config
if len(mcp_servers_config) > 0:
return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable)
if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or engine_context['engine_tools']:
return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context)
return None
@ -332,6 +328,7 @@ class BaseChatStep(IChatStep):
no_references_setting, problem_text, mcp_enable, mcp_tool_ids, mcp_servers, mcp_source, tool_enable, tool_ids,
mcp_output_enable)
chat_record_id = uuid.uuid7()
self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id)
r = StreamingHttpResponse(
streaming_content=event_content(chat_result, chat_id, chat_record_id, paragraph_list,
post_response_handler, manage, self, chat_model, message_list, problem_text,
@ -375,7 +372,8 @@ class BaseChatStep(IChatStep):
chat_model, message_list,
)
if mcp_result:
return mcp_result, True
from application.flow.tools import collect_mcp_response
return collect_mcp_response(mcp_result), True
return chat_model.invoke(message_list), True
def execute_block(self, message_list: List[BaseMessage],
@ -401,6 +399,7 @@ class BaseChatStep(IChatStep):
reasoning = Reasoning(reasoning_content_start,
reasoning_content_end)
chat_record_id = uuid.uuid7()
self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id)
# 调用模型
try:
chat_result, is_ai_chat = self.get_block_result(message_list, chat_model, paragraph_list,

View File

@ -1,3 +1,4 @@
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
# coding=utf-8
"""
@project: maxkb
@ -60,14 +61,19 @@ class BaseSearchDatasetStep(ISearchDatasetStep):
knowledge_id_list)
if len(knowledge_id_list) == 0:
return []
from knowledge.versions import resolve_search
knowledge_id_list = resolve_search(knowledge_id_list, manage.context.get('knowledge_version_map'), workspace_id)
# Exclusion IDs must come from the selected content, including frozen inactive documents.
from knowledge.models import Document
exclude_document_id_list = list(Document.objects.filter(knowledge_id__in=knowledge_id_list,
is_active=False).values_list('id', flat=True))
exec_problem_text = padding_problem_text if padding_problem_text is not None else problem_text
model_id = get_embedding_id(knowledge_id_list)
model = get_model_by_id(model_id, workspace_id)
if model.model_type != "EMBEDDING":
raise Exception(_("Model does not exist"))
self.context['model_name'] = model.name
default_params = get_model_default_params(model)
embedding_model = ModelManage.get_model(model_id, lambda _id: get_model(model, **{**default_params}))
embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list)
embedding_value = embedding_model.embed_query(exec_problem_text)
vector = VectorStore.get_embedding_vector()
embedding_list = vector.query(exec_problem_text, embedding_value, knowledge_id_list, None, exclude_document_id_list,

View File

@ -21,6 +21,7 @@ from .loop_continue_node import BaseLoopContinueNode
from .loop_node import *
from .loop_start_node import *
from .mcp_node import BaseMcpNode
from .component_node import BaseComponentNode
from .parameter_extraction_node import BaseParameterExtractionNode
from .question_node import *
from .reranker_node import *
@ -42,7 +43,7 @@ node_list = [BaseStartStepNode, BaseChatNode, BaseSearchKnowledgeNode, BaseSearc
BaseToolNodeNode, BaseToolLibNodeNode, BaseRerankerNode, BaseApplicationNode,
BaseDocumentExtractNode,
BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,
BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseTextToVideoNode, BaseImageToVideoNode,
BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseComponentNode, BaseTextToVideoNode, BaseImageToVideoNode,
BaseVideoUnderstandNode,
BaseIntentNode, BaseLoopNode, BaseLoopStartStepNode,
BaseLoopContinueNode,

View File

@ -206,29 +206,23 @@ class BaseChatNode(IChatNode):
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_id, mcp_tool_ids, tool_ids,
mcp_output_enable, chat_model, message_list, history_message, question):
if not mcp_enable and not tool_enable:
enabled = (self.node_params.get('model_setting') or {}).get('engine_tools', [])
if not mcp_enable and not tool_enable and not enabled:
return None
mcp_servers_config = {}
# 迁移过来mcp_source是None
if mcp_source is None:
mcp_source = 'custom'
if mcp_enable:
# 兼容老数据
if not mcp_tool_ids:
mcp_tool_ids = []
if mcp_tool_id:
mcp_tool_ids = list(set(mcp_tool_ids + [mcp_tool_id]))
if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers:
mcp_servers_config = json.loads(mcp_servers)
mcp_servers_config = self.handle_variables(mcp_servers_config)
elif mcp_tool_ids:
mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values()
for mcp_tool in mcp_tools:
if mcp_tool and mcp_tool['is_active']:
mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])}
mcp_servers_config = self.handle_variables(mcp_servers_config)
from system_manage.mcp_agent import context_for
ids = list(mcp_tool_ids or [])
if mcp_tool_id: ids.append(mcp_tool_id)
body = dict(self.workflow_manage.get_body())
# Explicit AI-node selection is limited to knowledge bases associated with this workflow.
from system_manage.mcp_workflows import knowledge_scope
body['knowledge_id_list'] = knowledge_scope(self.workflow_manage.flow)
engine_context = context_for(body, engine_tools=enabled,
tool_ids=ids if mcp_enable and mcp_source not in (None, 'custom') else [],
custom=self.handle_variables(json.loads(mcp_servers) if isinstance(mcp_servers, str) else mcp_servers) if mcp_enable and mcp_source in (None, 'custom') and mcp_servers else None,
transform=self.handle_variables, node_id=self.id)
if tool_enable:
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
@ -248,8 +242,8 @@ class BaseChatNode(IChatNode):
self.context['execute_ids'].append(_id)
mcp_servers_config[str(tool.id)] = tool_config
if len(mcp_servers_config) > 0:
r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable)
if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or enabled:
r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context)
return NodeResult(
{'result': r, 'chat_model': chat_model, 'message_list': message_list,
'history_message': [{'content': message.content, 'role': message.type} for message in
@ -315,7 +309,9 @@ class BaseChatNode(IChatNode):
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
if os.path.exists(code_path):
os.remove(code_path)
from system_manage.mcp_agent import call_records
return {
'engine_calls': call_records(self.workflow_manage.get_body(), self.id),
'name': self.node.properties.get('stepName'),
"index": index,
'run_time': self.context.get('run_time'),

View File

@ -0,0 +1 @@
from .base_component_node import BaseComponentNode

View File

@ -0,0 +1,64 @@
from copy import deepcopy
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from application.flow.i_step_node import INode, NodeResult
from system_manage import mcp_runtime
class ComponentParams(serializers.Serializer):
component_id = serializers.CharField()
parameters = serializers.DictField(default=dict)
sources = serializers.DictField(default=dict)
timeout = serializers.IntegerField(default=60, min_value=1, max_value=600)
retries = serializers.IntegerField(default=0, min_value=0, max_value=2)
priority = serializers.IntegerField(default=5, min_value=0, max_value=9)
def workflow_identity(workflow):
from application.models import Application
from users.models import User
body = workflow.get_body()
app = Application.objects.filter(id=body['application_id'], workspace_id=body['workspace_id']).first()
if not app or not app.user_id:
raise ValidationError('应用不存在或没有可用的所有者')
user = User.objects.get(id=body.get('engine_user_id') or app.user_id, is_active=True)
return body, user
class BaseComponentNode(INode):
type = 'component-node'
def get_node_params_serializer_class(self):
return ComponentParams
def _run(self):
return self.execute(**self.node_params_serializer.data)
def execute(self, component_id, parameters=None, sources=None, timeout=60, retries=0, priority=5, **kwargs):
body, user = workflow_identity(self.workflow_manage)
def render(value):
if isinstance(value, str): return self.workflow_manage.generate_prompt(value)
if isinstance(value, dict): return {key: render(item) for key, item in value.items()}
if isinstance(value, list): return [render(item) for item in value]
return value
params = deepcopy(parameters or {})
for key, value in params.items():
if (sources or {}).get(key) == 'reference':
if not isinstance(value, list) or len(value) < 2:
raise ValidationError('请选择有效的上游变量')
params[key] = self.workflow_manage.get_reference_field(value[0], value[1:])
else:
params[key] = render(value)
result = mcp_runtime.invoke(body['workspace_id'], user, component_id, params,
parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'),
scope={**body, 'node_id': self.id, 'source': 'workflow-component'},
timeout=timeout, retries=retries, priority=priority)
return NodeResult(result, {})
def save_context(self, details, workflow_manage):
self.context.update({key: details.get(key) for key in ('text', 'structured', 'result', 'task_id', 'component_id')})
def get_details(self, index, **kwargs):
return {'name': self.node.properties.get('stepName'), 'index': index, 'type': self.type,
'status': self.status, 'err_message': self.err_message, 'run_time': self.context.get('run_time'),
**{key: self.context.get(key) for key in ('component_id', 'text', 'structured', 'task_id')}}

View File

@ -9,12 +9,16 @@ from application.flow.i_step_node import INode, NodeResult
class McpNodeSerializer(serializers.Serializer):
mcp_servers = serializers.JSONField(required=True, label=_("Mcp servers"))
mcp_servers = serializers.JSONField(required=False, default=dict, label=_("Mcp servers"))
mcp_server = serializers.CharField(required=True, label=_("Mcp server"))
mcp_tool = serializers.CharField(required=True, label=_("Mcp tool"))
mcp_tool_id = serializers.CharField(required=False, label=_("Mcp tool"), allow_null=True, allow_blank=True)
mcp_source = serializers.CharField(required=False, label=_("Mcp source"), allow_blank=True, allow_null=True)
tool_params = serializers.DictField(required=True, label=_("Tool parameters"))
tool_form_field = serializers.ListField(required=False, default=list)
timeout = serializers.IntegerField(default=60, min_value=1, max_value=600)
retries = serializers.IntegerField(default=0, min_value=0, max_value=2)
priority = serializers.IntegerField(default=5, min_value=0, max_value=9)
class IMcpNode(INode):

View File

@ -1,75 +1,59 @@
# coding=utf-8
import asyncio
import json
from copy import deepcopy
from typing import List
from django.db.models import QuerySet
from langchain_mcp_adapters.client import MultiServerMCPClient
from rest_framework.exceptions import ValidationError
from application.flow.i_step_node import NodeResult
from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode
from tools.models import Tool
from application.flow.step_node.component_node.base_component_node import workflow_identity
from system_manage import mcp_runtime
class BaseMcpNode(IMcpNode):
def save_context(self, details, workflow_manage):
self.context['result'] = details.get('result')
self.context['tool_params'] = details.get('tool_params')
self.context['mcp_tool'] = details.get('mcp_tool')
self.context.update({key: details.get(key) for key in ('result', 'text', 'structured', 'tool_params', 'mcp_tool', 'mcp_server', 'attempts', 'task_id')})
def execute(self, mcp_servers, mcp_server, mcp_tool, mcp_tool_id, mcp_source, tool_params,**kwargs) -> NodeResult:
if mcp_source == 'referencing':
if not mcp_tool_id:
raise ValueError("MCP tool ID is required when mcp_source is 'referencing'.")
tool = QuerySet(Tool).filter(id=mcp_tool_id).first()
if not tool:
raise ValueError(f"Tool with ID {mcp_tool_id} not found.")
if not tool.is_active:
raise ValueError(f"Tool with ID {mcp_tool_id} is inactive.")
servers = json.loads(tool.code)
servers = self.handle_variables(servers) # 处理servers中的变量
params = json.loads(json.dumps(tool_params))
params = self.handle_variables(params)
def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, mcp_tool_id=None, mcp_source=None,
tool_form_field=None, timeout=60, retries=0, priority=5, **kwargs):
body, user = workflow_identity(self.workflow_manage)
params = self.handle_variables(deepcopy(tool_params), tool_form_field or [])
key = 'mcp:' + str(mcp_tool_id) if mcp_source == 'referencing' else 'mcp-custom'
result = mcp_runtime.invoke(body['workspace_id'], user, key,
{'servers': self.handle_variables(mcp_runtime.connections(mcp_servers)) if key == 'mcp-custom' else None,
'server': mcp_server, 'tool': mcp_tool, 'arguments': params},
parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'),
scope={**body, 'node_id': self.id, 'source': 'workflow-mcp'}, timeout=timeout, retries=retries, priority=priority)
return NodeResult({**result, 'tool_params': params}, {})
def handle_variables(self, value, fields=None):
if isinstance(value, dict):
output = {}
for key, item in value.items():
field = next((f for f in fields or [] if f.get('field') == key), {})
if field.get('source') == 'custom' and field.get('input_type') == 'JsonInput' and isinstance(item, str):
try: output[key] = json.loads(self.workflow_manage.generate_prompt(item))
except ValueError: raise ValidationError('工具参数必须为有效 JSON') from None
elif field.get('source') == 'custom':
output[key] = self.workflow_manage.generate_prompt(item) if isinstance(item, str) else item
elif field.get('source') == 'referencing':
if not isinstance(item, list) or len(item) < 2: raise ValidationError('请选择工具参数的上游变量')
output[key] = self.get_reference_content(item)
else:
servers = json.loads(mcp_servers)
servers = self.handle_variables(servers) # 处理servers中的变量
params = json.loads(json.dumps(tool_params))
params = self.handle_variables(params)
async def call_tool(t, a):
client = MultiServerMCPClient(servers)
async with client.session(mcp_server) as s:
return await s.call_tool(t, a)
res = asyncio.run(call_tool(mcp_tool, params))
return NodeResult(
{'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {})
def handle_variables(self, tool_params):
# 处理参数中的变量
for k, v in tool_params.items():
if type(v) == str:
tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k])
if type(v) == dict:
self.handle_variables(v)
if (type(v) == list) and (type(v[0]) == str):
tool_params[k] = self.get_reference_content(v)
return tool_params
output[key] = self.handle_variables(item, fields)
return output
if isinstance(value, str): return self.workflow_manage.generate_prompt(value)
if isinstance(value, list):
known = set(self.workflow_manage.flow.node_map) | {'global', 'chat', 'loop'}
if len(value) >= 2 and all(isinstance(v, str) for v in value) and value[0] in known:
return self.get_reference_content(value)
return [self.handle_variables(item, fields) for item in value]
return value
def get_reference_content(self, fields: List[str]):
return self.workflow_manage.get_reference_field(
fields[0],
fields[1:])
return self.workflow_manage.get_reference_field(fields[0], fields[1:])
def get_details(self, index: int, **kwargs):
return {
'name': self.node.properties.get('stepName'),
"index": index,
'run_time': self.context.get('run_time'),
'status': self.status,
'err_message': self.err_message,
'type': self.node.type,
'mcp_tool': self.context.get('mcp_tool'),
'tool_params': self.context.get('tool_params'),
'result': self.context.get('result'),
}
return {'name': self.node.properties.get('stepName'), 'index': index,
'run_time': self.context.get('run_time'), 'status': self.status,
'err_message': self.err_message, 'type': self.node.type,
**{key: self.context.get(key) for key in ('mcp_tool', 'mcp_server', 'tool_params', 'result', 'text', 'structured', 'attempts', 'task_id')}}

View File

@ -18,6 +18,7 @@ from common.utils.common import flat_map
class DatasetSettingSerializer(serializers.Serializer):
version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False)
# 需要查询的条数
top_n = serializers.IntegerField(required=True,
label=_("Reference segment number"))

View File

@ -1,3 +1,4 @@
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
# coding=utf-8
"""
@project: maxkb
@ -100,9 +101,17 @@ class BaseSearchKnowledgeNode(ISearchKnowledgeStepNode):
knowledge_id_list)
if len(knowledge_id_list) == 0:
return get_none_result(question)
from knowledge.versions import resolve_search, resolve_document_scope
knowledge_id_list = resolve_search(knowledge_id_list,
self.workflow_manage.get_body().get('knowledge_version_map'),
self.workflow_manage.get_body().get('workspace_id'))
document_id_list = resolve_document_scope(document_id_list,
self.workflow_manage.get_body().get('knowledge_version_map'),
self.workflow_manage.get_body().get('workspace_id'))
model_id = get_embedding_id(knowledge_id_list)
workspace_id = self.workflow_manage.get_body().get('workspace_id')
embedding_model = get_model_instance_by_model_workspace_id(model_id, workspace_id)
get_model_instance_by_model_workspace_id(model_id, workspace_id) # Preserve resource access validation.
embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list)
embedding_value = embedding_model.embed_query(question)
vector = VectorStore.get_embedding_vector()
exclude_document_id_list = [str(document.id) for document in

View File

@ -13,7 +13,7 @@ import threading
from typing import Iterator
from django.http import StreamingHttpResponse
from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk
from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk, AIMessage, SystemMessage
from langchain_mcp_adapters.client import MultiServerMCPClient
from langgraph.prebuilt import create_react_agent
from application.flow.i_step_node import WorkFlowPostHandler
@ -267,17 +267,23 @@ def get_global_loop():
return _global_loop
async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True):
client = MultiServerMCPClient(json.loads(mcp_servers))
tools = await client.get_tools()
async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None):
servers = json.loads(mcp_servers)
tools = await MultiServerMCPClient(servers).get_tools() if servers else []
if engine_context:
from system_manage.mcp_agent import tools as engine_tools
tools += await engine_tools(engine_context)
agent = create_react_agent(chat_model, tools)
response = agent.astream({"messages": message_list}, stream_mode='messages')
messages = message_list
if not any(isinstance(message, SystemMessage) for message in message_list):
messages = [SystemMessage(content='你是应用助手。根据用户请求选择已授权工具。需要外部数据或生成图谱时实际调用工具,不要编造工具结果。工具执行完成后依据结果回答,保存导图后返回打开链接,避免重复调用。'), *message_list]
response = agent.astream({"messages": messages}, stream_mode='messages')
# 用于存储工具调用信息
tool_calls_info = {}
async for chunk in response:
if isinstance(chunk[0], AIMessageChunk):
if isinstance(chunk[0], (AIMessageChunk, AIMessage)):
tool_calls = chunk[0].additional_kwargs.get('tool_calls', [])
for tool_call in tool_calls:
tool_id = tool_call.get('id', '')
@ -287,6 +293,10 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
'name': tool_call.get('function', {}).get('name', ''),
'input': tool_call.get('function', {}).get('arguments', '')
}
for tool_call in chunk[0].tool_calls:
if tool_call.get('id'):
tool_calls_info[tool_call['id']] = {'name': tool_call['name'],
'input': json.dumps(tool_call.get('args', {}), ensure_ascii=False)}
yield chunk[0]
if mcp_output_enable and isinstance(chunk[0], ToolMessage):
@ -303,14 +313,30 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
yield chunk[0]
def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True):
def collect_mcp_response(response):
"""Use the same tool stream for the non-streaming chat API."""
from langchain_core.messages import AIMessage
parts, reasoning, metadata = [], [], {}
for chunk in response:
if isinstance(chunk.content, str):
parts.append(chunk.content)
else:
parts.extend(item.get('text', '') for item in chunk.content if isinstance(item, dict))
metadata.update(chunk.response_metadata)
value = chunk.additional_kwargs.get('reasoning_content')
if value: reasoning.append(value)
if reasoning: metadata['reasoning_content'] = ''.join(reasoning)
return AIMessage(content=''.join(parts), response_metadata=metadata)
def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None):
"""使用全局事件循环,不创建新实例"""
result_queue = queue.Queue()
loop = get_global_loop() # 使用共享循环
async def _run():
try:
async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable)
async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable, engine_context)
async for chunk in async_gen:
result_queue.put(('data', chunk))
except Exception as e:

View File

@ -98,6 +98,7 @@ class NoReferencesSetting(serializers.Serializer):
class KnowledgeSettingSerializer(serializers.Serializer):
version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False)
top_n = serializers.FloatField(required=True, max_value=10000, min_value=1,
label=_("Reference segment number"))
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
@ -132,6 +133,8 @@ class ModelKnowledgeAssociation(serializers.Serializer):
if not QuerySet(Model).filter(id=model_id).exists():
raise AppApiException(500, f'{_("Model does not exist")}【{model_id}】')
knowledge_id_list = list(set(self.data.get('knowledge_id_list', [])))
if Knowledge.objects.filter(id__in=knowledge_id_list, meta__has_key='_version_snapshot').exists():
raise AppApiException(500, '请关联原知识库,再选择历史版本')
if workspace_id is not None and len(workspace_id) > 0:
exist_knowledge_id_list = [
str(knowledge.get('id')) for knowledge in KnowledgeSerializer.Query(data={
@ -148,6 +151,8 @@ class ModelKnowledgeAssociation(serializers.Serializer):
class ModelSettingSerializer(serializers.Serializer):
engine_tools = serializers.ListField(required=False, child=serializers.ChoiceField(
choices=['knowledge-search', 'mindmap-save']), max_length=2)
prompt = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
label=_("Prompt word"))
system = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
@ -272,6 +277,9 @@ class ApplicationCreateSerializer(serializers.Serializer):
ModelKnowledgeAssociation(data={'user_id': user_id, 'workspace_id': workspace_id,
'model_id': self.data.get('model_id'),
'knowledge_id_list': self.data.get('knowledge_id_list')}).is_valid()
from knowledge.versions import validate_selection
validate_selection((self.data.get('knowledge_setting') or {}).get('version_selection', {}),
workspace_id, self.data.get('knowledge_id_list') or [])
@staticmethod
def to_application_model(user_id: str, workspace_id: str, application: Dict):
@ -671,21 +679,8 @@ class ApplicationOperateSerializer(serializers.Serializer):
if with_valid:
self.is_valid(raise_exception=True)
McpServersSerializer(data=instance).is_valid(raise_exception=True)
servers = json.loads(instance.get('mcp_servers'))
for server, config in servers.items():
if config.get('transport') not in ['sse', 'streamable_http']:
raise AppApiException(500, _('Only support transport=sse or transport=streamable_http'))
tools = []
for server in servers:
tools += [
{
'server': server,
'name': tool.name,
'description': tool.description,
'args_schema': tool.args_schema,
}
for tool in asyncio.run(get_mcp_tools({server: servers[server]}))]
return tools
from system_manage.mcp_runtime import discover
return discover(instance.get('mcp_servers'))
def delete(self, with_valid=True):
if with_valid:
@ -780,6 +775,13 @@ class ApplicationOperateSerializer(serializers.Serializer):
publish_user_name=user.username,
workspace_id=workspace_id)
self.reset_application_version(work_flow_version, application)
from knowledge.versions import application_selection, validate_selection
root_ids = [str(k) for k in ApplicationKnowledgeMapping.objects.filter(application=application)
.values_list('knowledge_id', flat=True)]
selected, flow_ids = application_selection(application.knowledge_setting,
application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW else None)
validate_selection(selected, workspace_id, root_ids + flow_ids)
work_flow_version.knowledge_setting = {**(work_flow_version.knowledge_setting or {}), '_knowledge_ids': root_ids}
work_flow_version.save()
access_token = hashlib.md5(
str(uuid.uuid7()).encode()).hexdigest()[
@ -838,6 +840,13 @@ class ApplicationOperateSerializer(serializers.Serializer):
application_id = self.data.get("application_id")
application = QuerySet(Application).get(id=application_id)
from knowledge.versions import validate_selection, application_selection
selected_ids = instance.get('knowledge_id_list')
if selected_ids is None:
selected_ids = list(ApplicationKnowledgeMapping.objects.filter(application_id=application_id).values_list('knowledge_id', flat=True))
selected, flow_ids = application_selection(instance.get('knowledge_setting', application.knowledge_setting),
instance.get('work_flow', application.work_flow) if application.type == ApplicationTypeChoices.WORK_FLOW else None)
validate_selection(selected, application.workspace_id, list(selected_ids) + flow_ids)
if instance.get('model_id') is None or len(instance.get('model_id')) == 0:
application.model_id = None
else:

View File

@ -311,6 +311,9 @@ class ApplicationChatRecordAddKnowledgeSerializer(serializers.Serializer):
if with_valid:
ApplicationChatRecordAddKnowledgeSerializer(data=instance).is_valid(raise_exception=True)
self.is_valid(raise_exception=True)
# This endpoint receives its knowledge ID in the body rather than the route.
from knowledge.versions import root_knowledge
root_knowledge(self.data.get('workspace_id') or 'default', self.data['knowledge_id'])
if scope == 'WORKSPACE':
is_permission = get_is_permissions(request=request, workspace_id=self.data.get('workspace_id'),
knowledge_id=self.data.get("knowledge_id"))(

View File

@ -56,6 +56,7 @@ class ChatInfo:
self.application = None
self.chat_user = None
self.debug = debug
self.knowledge_version_map = None
@staticmethod
def get_no_references_setting(knowledge_setting, model_setting):
@ -75,14 +76,14 @@ class ChatInfo:
raise ChatException(500, _('The application does not exist'))
else:
application = QuerySet(ApplicationVersion).filter(application_id=self.application_id).order_by(
'-create_time')[0:1].first()
'-create_time', '-id')[0:1].first()
if not application:
raise ChatException(500, _("The application has not been published. Please use it after publishing."))
if application.type == ApplicationTypeChoices.SIMPLE.value:
# 数据集id列表
knowledge_id_list = [str(row.knowledge_id) for row in
QuerySet(ApplicationKnowledgeMapping).filter(
application_id=self.application_id)]
published_ids = (application.knowledge_setting or {}).get('_knowledge_ids') if not self.debug else None
knowledge_id_list = published_ids if published_ids is not None else [str(row.knowledge_id) for row in
QuerySet(ApplicationKnowledgeMapping).filter(application_id=self.application_id)]
# 需要排除的文档
exclude_document_id_list = [str(document.id) for document in
@ -92,6 +93,16 @@ class ChatInfo:
self.knowledge_id_list = knowledge_id_list
self.exclude_document_id_list = exclude_document_id_list
self.application = application
from knowledge.versions import pin_selection, validate_selection, application_selection
selection, flow_ids = application_selection(application.knowledge_setting,
application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW.value else None)
version_knowledge_ids = list(self.knowledge_id_list)
version_knowledge_ids.extend(flow_ids)
if self.knowledge_version_map is None:
validate_selection(selection, application.workspace_id, version_knowledge_ids)
self.knowledge_version_map = pin_selection(selection, application.workspace_id, self.chat_id)
elif self.knowledge_version_map:
pin_selection({}, application.workspace_id, self.chat_id, self.knowledge_version_map)
return application
def get_chat_user(self, asker=None):
@ -155,6 +166,7 @@ class ChatInfo:
model_params_setting = credential.get_model_params_setting_form(model.model_name).get_default_form_data()
return {
'knowledge_id_list': self.knowledge_id_list,
'knowledge_version_map': self.knowledge_version_map or {},
'exclude_document_id_list': self.exclude_document_id_list,
'exclude_paragraph_id_list': [],
'top_n': 3 if knowledge_setting.get('top_n') is None else knowledge_setting.get('top_n'),
@ -204,6 +216,7 @@ class ChatInfo:
if not self.debug:
if not QuerySet(Chat).filter(id=self.chat_id).exists():
Chat(id=self.chat_id, application_id=self.application_id, abstract=question[0:1024],
meta={'knowledge_versions': self.knowledge_version_map or {}},
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
asker=self.get_chat_user()).save()
@ -244,6 +257,7 @@ class ChatInfo:
if not self.debug:
if not QuerySet(Chat).filter(id=self.chat_id).exists():
Chat(id=self.chat_id, application_id=self.application_id, abstract=chat_record.problem_text[0:1024],
meta={'knowledge_versions': self.knowledge_version_map or {}},
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
asker=self.get_chat_user()).save()
else:
@ -260,6 +274,7 @@ class ChatInfo:
'chat_user_type': self.chat_user_type,
'knowledge_id_list': self.knowledge_id_list,
'exclude_document_id_list': self.exclude_document_id_list,
'knowledge_version_map': self.knowledge_version_map,
'application_id': self.application_id,
'chat_record_list': [self.chat_record_to_map(c) for c in self.chat_record_list][-20:],
'debug': self.debug
@ -309,6 +324,7 @@ class ChatInfo:
chat_info_dict.get('application_id'),
debug=chat_info_dict.get('debug'))
c.chat_record_list = [ChatInfo.map_to_chat_record(c_r) for c_r in chat_info_dict.get('chat_record_list')]
c.knowledge_version_map = chat_info_dict.get('knowledge_version_map')
return c
@staticmethod

View File

@ -16,6 +16,6 @@ from (select application."id"::text, application."name",
from application
left join "user" on user_id = "user".id
where application."id"::text in (select target
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
from effective_resource_permission ${workspace_user_resource_permission_query_set}
and 'VIEW' = any (permission_list))) temp
${application_query_set}

View File

@ -16,7 +16,7 @@ from (select application."id"::text, application."name",
from application
left join "user" on user_id = "user".id
where "application".id::text in (select target
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
from effective_resource_permission ${workspace_user_resource_permission_query_set}
and case
when auth_type = 'ROLE' then
'ROLE' = any (permission_list)
@ -28,9 +28,9 @@ from (select application."id"::text, application."name",
from role_permission role_permission
right join user_role_relation user_role_relation
on user_role_relation.role_id = role_permission.role_id
where user_role_relation.user_id = workspace_user_resource_permission.user_id
where user_role_relation.user_id = effective_resource_permission.user_id
and user_role_relation.workspace_id =
workspace_user_resource_permission.workspace_id)
effective_resource_permission.workspace_id)
else
'VIEW' = any (permission_list)

View File

@ -272,6 +272,8 @@ class McpServers(APIView):
CompareConstants.AND),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
def post(self, request: Request, workspace_id, application_id: str):
from system_manage.access_control import enforce
enforce(request, 'application', application_id)
return result.success(ApplicationOperateSerializer(
data={'mcp_servers': request.query_params.get('mcp_servers'), 'workspace_id': workspace_id,
'user_id': request.user.id,

View File

@ -132,9 +132,11 @@ class OpenView(APIView):
CompareConstants.AND),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
def get(self, request: Request, workspace_id: str, application_id: str):
from system_manage.access_control import enforce
enforce(request, 'application', application_id)
return result.success(OpenChatSerializers(
data={'workspace_id': workspace_id, 'application_id': application_id,
'chat_user_id': str(uuid.uuid7()), 'chat_user_type': ChatUserType.ANONYMOUS_USER,
'chat_user_id': str(request.user.id), 'chat_user_type': ChatUserType.PLATFORM_USER.value,
'debug': True}).open())
@ -152,7 +154,7 @@ class ChatView(APIView):
tags=[_('Application')] # type: ignore
)
def post(self, request: Request, chat_id: str):
return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data)
return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data, request=request)
class PromptGenerateView(APIView):
authentication_classes = [TokenAuth]

View File

@ -131,10 +131,12 @@ def get_post_handler(chat_info: ChatInfo):
class DebugChatSerializers(serializers.Serializer):
chat_id = serializers.UUIDField(required=True, label=_("Conversation ID"))
def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse()):
def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse(), request=None):
self.is_valid(raise_exception=True)
chat_id = self.data.get('chat_id')
chat_info: ChatInfo = ChatInfo.get_cache(chat_id)
from system_manage.access_control import enforce
enforce(request, 'application', chat_info.application_id)
application = QuerySet(Application).filter(id=chat_info.application_id).first()
chat_info.application = application
return ChatSerializers(data={
@ -404,7 +406,8 @@ class ChatSerializers(serializers.Serializer):
'workspace_id': workspace_id,
'debug': debug,
'chat_user': chat_info.get_chat_user(),
'application_id': str(chat_info.application_id)},
'application_id': str(chat_info.application_id),
'knowledge_version_map': chat_info.knowledge_version_map or {}},
WorkFlowPostHandler(chat_info),
base_to_response, form_data, image_list, document_list, audio_list,
video_list,
@ -481,6 +484,9 @@ class ChatSerializers(serializers.Serializer):
is_active=False)]
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), knowledge_id_list,
exclude_document_id_list, application.id)
previous = Chat.objects.filter(pk=chat_id).first()
if previous and 'knowledge_versions' in (previous.meta or {}):
chat_info.knowledge_version_map = previous.meta['knowledge_versions']
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
chat_record_list.sort(key=lambda r: r.create_time)
for chat_record in chat_record_list:
@ -490,6 +496,9 @@ class ChatSerializers(serializers.Serializer):
def re_open_chat_work_flow(self, chat_id, application):
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), [], [],
application.id)
previous = Chat.objects.filter(pk=chat_id).first()
if previous and 'knowledge_versions' in (previous.meta or {}):
chat_info.knowledge_version_map = previous.meta['knowledge_versions']
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
chat_record_list.sort(key=lambda r: r.create_time)
for chat_record in chat_record_list:

View File

@ -86,6 +86,8 @@ class AnonymousAuthenticationSerializer(serializers.Serializer):
access_token = self.data.get("access_token")
application_access_token = QuerySet(ApplicationAccessToken).filter(access_token=access_token).first()
if application_access_token is not None and application_access_token.is_active:
from system_manage.access_control import enforce
enforce(request, 'application', application_access_token.application_id)
api_key = self.data.get('api_key') or self.get_request_api_key(request)
if api_key:
application_api_key = self.validate_application_api_key(application_access_token.application_id, api_key)
@ -121,6 +123,8 @@ class AuthProfileSerializer(serializers.Serializer):
if not application_access_token.is_active:
raise NotFound404(404, _("Invalid access_token"))
application_id = application_access_token.application_id
from system_manage.access_control import enforce
enforce(request, 'application', application_id)
api_key = self.data.get("api_key") or AnonymousAuthenticationSerializer.get_request_api_key(request)
if api_key:
AnonymousAuthenticationSerializer.validate_application_api_key(application_id, api_key)

View File

@ -85,7 +85,19 @@ class TokenAuth(TokenAuthentication):
token_details = TokenDetails(token)
for handle in handles:
if handle.support(request, token, token_details.get_token_details):
return handle.handle(request, token, token_details.get_token_details)
authenticated = handle.handle(request, token, token_details.get_token_details)
from knowledge.versions import guard_snapshot_request
guard_snapshot_request(request)
from common.constants.permission_constants import ChatAuth
if isinstance(authenticated[1], ChatAuth):
from system_manage.access_control import enforce
match = getattr(request, 'resolver_match', None)
routed_id = match.kwargs.get('application_id') if match else None
if routed_id and str(routed_id) != str(authenticated[1].application_id):
from common.exception.app_exception import AppUnauthorizedFailed
raise AppUnauthorizedFailed(403, '无权访问该助手')
enforce(request, 'application', authenticated[1].application_id)
return authenticated
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect! illegal user'))
except Exception as e:
traceback.print_stack()

View File

@ -275,6 +275,10 @@ def get_role_list(user,
def get_auth(user):
from system_manage.platform_access import is_admin
# The old field is a compatibility projection, never an authority.
administrator = is_admin(user)
user.role = 'ADMIN' if administrator else 'USER'
workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping")
workspace_model = DatabaseModelManage.get_model("workspace_model")
role_model = DatabaseModelManage.get_model("role_model")
@ -284,7 +288,15 @@ def get_auth(user):
role_model, role_permission_mapping_model)
role_list = get_role_list(user, workspace_user_role_mapping_model, workspace_model,
role_model, role_permission_mapping_model)
return Auth(role_list, permission_list)
if workspace_user_role_mapping_model is None:
role_list = ['ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')] if administrator else ['USER', get_role_permission(RoleConstants.USER, 'default')]
elif administrator:
role_list = list(set(role_list) | {'ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')})
else:
role_list = [role for role in role_list if role not in ('ADMIN', 'EXTENDS_ADMIN')]
from system_manage.platform_access import permission_strings
# Evaluate new grants live, independently of legacy cached permissions.
return Auth(role_list, list(set(permission_list) | permission_strings(user)))
class UserToken(AuthBaseHandle):
@ -306,4 +318,6 @@ class UserToken(AuthBaseHandle):
if not user.is_active or user.password != cache_token.password:
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect'))
auth = get_auth(user)
from system_manage.platform_access import check_feature_request
check_feature_request(request, user)
return user, auth

View File

@ -8,6 +8,9 @@
"""
import os
import logging
import threading
from pathlib import Path
class MKTokenizer:
def __init__(self, tokenizer):
@ -17,14 +20,38 @@ class MKTokenizer:
return self.tokenizer.encode(text).ids
class ByteTokenEstimate:
"""Conservative token budget when an offline tokenizer is unavailable.
Provider-reported usage remains authoritative. This only supplies local
estimates for prompt budgeting; it never downloads model files.
"""
def encode(self, text):
return list(text.encode('utf-8'))
class TokenizerManage:
tokenizer = None
_lock = threading.Lock()
@staticmethod
def get_tokenizer():
if TokenizerManage.tokenizer is not None:
return TokenizerManage.tokenizer
with TokenizerManage._lock:
if TokenizerManage.tokenizer is not None:
return TokenizerManage.tokenizer
from tokenizers import Tokenizer
# 创建Tokenizer
model_path = os.path.join("/opt/maxkb-app", "model", "tokenizer", "models--bert-base-cased")
with open(f"{model_path}/refs/main", encoding="utf-8") as f: snapshot = f.read()
TokenizerManage.tokenizer = Tokenizer.from_file(f"{model_path}/snapshots/{snapshot}/tokenizer.json")
return MKTokenizer(TokenizerManage.tokenizer)
model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or
'/opt/maxkb-app/model/tokenizer/models--bert-base-cased')
try:
path = model_path if model_path.is_file() else model_path / 'tokenizer.json'
if not path.is_file():
snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip()
path = model_path / 'snapshots' / snapshot / 'tokenizer.json'
TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path)))
except Exception:
# A missing/corrupt tokenizer must not prevent remote-model chat.
logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准')
TokenizerManage.tokenizer = ByteTokenEstimate()
return TokenizerManage.tokenizer

View File

@ -357,7 +357,7 @@ class ListenerManagement:
@staticmethod
def update_problem(args: UpdateProblemArgs):
problem_paragraph_mapping_list = QuerySet(ProblemParagraphMapping).filter(problem_id=args.problem_id)
embed_value = args.embedding_model.embed_query(args.problem_content)
embed_value = args.embedding_model.embed_documents([args.problem_content])[0]
VectorStore.get_embedding_vector().update_by_source_ids([v.id for v in problem_paragraph_mapping_list],
{'embedding': embed_value})

View File

@ -10,7 +10,6 @@ import logging
import traceback
from rest_framework.exceptions import ValidationError, ErrorDetail, APIException
from rest_framework.utils.serializer_helpers import ReturnDict
from rest_framework.views import exception_handler
from common import result
@ -60,28 +59,18 @@ def validation_error_to_result(exc: ValidationError):
def find_err_detail(exc_detail):
if isinstance(exc_detail, ErrorDetail):
if isinstance(exc_detail, (ErrorDetail, str)):
return exc_detail
if isinstance(exc_detail, dict):
keys = exc_detail.keys()
for key in keys:
_label = get_label(key, exc_detail)
_value = exc_detail[key]
if isinstance(_value, list):
for v in _value:
r = find_err_detail(ReturnDict({key: v}, serializer=exc_detail.serializer))
if r is not None:
return r
if isinstance(_value, ErrorDetail):
return f"{_label}:{find_err_detail(_value)}"
if isinstance(_value, dict) and len(_value.keys()) > 0:
try:
return find_err_detail(ReturnDict(_value, serializer=exc_detail.serializer.fields[key]))
except Exception as e:
return _value
if isinstance(exc_detail, list):
for key, value in exc_detail.items():
detail = find_err_detail(value)
if detail is not None:
if key == 'non_field_errors':
return detail
return f'{get_label(key, exc_detail)}:{detail}'
if isinstance(exc_detail, (list, tuple)):
for v in exc_detail:
r = find_err_detail(ReturnDict(v, serializer=exc_detail.serializer.child))
r = find_err_detail(v)
if r is not None:
return r

View File

@ -0,0 +1,18 @@
import json
from django.test import SimpleTestCase
from rest_framework.exceptions import ValidationError
from common.exception.handle_exception import validation_error_to_result
class ValidationMessageTests(SimpleTestCase):
def test_plain_list_error_is_readable(self):
response = validation_error_to_result(ValidationError('最多可保存 10 张导图,请先删除不需要的导图'))
self.assertEqual(json.loads(response.content)['message'], '最多可保存 10 张导图,请先删除不需要的导图')
def test_nested_error_retains_field_context(self):
response = validation_error_to_result(ValidationError({'document': {'name': ['不能为空']}}))
self.assertEqual(json.loads(response.content)['message'], 'document:name:不能为空')
def test_non_field_error_has_no_internal_key(self):
response = validation_error_to_result(ValidationError({'non_field_errors': ['内容无效']}))
self.assertEqual(json.loads(response.content)['message'], '内容无效')

View File

@ -22,3 +22,4 @@ from .table_checkbox import *
from .radio_card_field import *
from .label import *
from .slider_field import *
from .switch_field import *

View File

@ -16,6 +16,7 @@ from charset_normalizer import detect
from common.handle.base_split_handle import BaseSplitHandle
from common.utils.logger import maxkb_logger
from common.utils.table_split import split_table_rows
def post_cell(cell_value):
@ -28,7 +29,7 @@ def row_to_md(row):
class CsvSplitHandle(BaseSplitHandle):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
buffer = get_buffer(file)
paragraphs = []
file_name = os.path.basename(file.name)
@ -46,22 +47,7 @@ class CsvSplitHandle(BaseSplitHandle):
return result
if len(title_row_list) == 0:
return result
result_item_content = ''
for row in reader:
next_md_content = row_to_md(row)
next_md_content_len = len(next_md_content)
result_item_content_len = len(result_item_content)
if len(result_item_content) == 0:
result_item_content += title_md_content
result_item_content += next_md_content
else:
if result_item_content_len + next_md_content_len < limit:
result_item_content += next_md_content
else:
paragraphs.append({'content': result_item_content, 'title': ''})
result_item_content = title_md_content + next_md_content
if len(result_item_content) > 0:
paragraphs.append({'content': result_item_content, 'title': ''})
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in reader), limit, overlap))
return result
except Exception as e:
maxkb_logger.error(f"Error processing CSV file {file.name}: {e}, {traceback.format_exc()}")

View File

@ -192,7 +192,7 @@ class DocSplitHandle(BaseSplitHandle):
for element
in elements])
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
file_name = os.path.basename(file.name)
try:
if type(limit) is str:
@ -206,9 +206,9 @@ class DocSplitHandle(BaseSplitHandle):
if len(image_list) > 0:
save_image(image_list)
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
except BaseException as e:
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")
return {

View File

@ -44,16 +44,16 @@ class HTMLSplitHandle(BaseSplitHandle):
return True
return False
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
buffer = get_buffer(file)
if type(limit) is str:
limit = int(limit)
if type(with_filter) is str:
with_filter = with_filter.lower() == 'true'
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
try:
encoding = get_encoding(buffer)
content = buffer.decode(encoding)

View File

@ -42,7 +42,7 @@ def check_links_in_pdf(doc):
class PdfSplitHandle(BaseSplitHandle):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
# 将上传的文件保存到临时文件中
for chunk in file.chunks():
@ -57,12 +57,12 @@ class PdfSplitHandle(BaseSplitHandle):
if type(with_filter) is str:
with_filter = with_filter.lower() == 'true'
# 处理有目录的pdf
result = self.handle_toc(pdf_document, limit)
result = self.handle_toc(pdf_document, limit, overlap)
if result is not None:
return {'name': file.name, 'content': result}
# 没目录但是有链接的pdf
result = self.handle_links(pdf_document, pattern_list, with_filter, limit)
result = self.handle_links(pdf_document, pattern_list, with_filter, limit, overlap)
if result is not None and len(result) > 0:
return {'name': file.name, 'content': result}
@ -70,9 +70,9 @@ class PdfSplitHandle(BaseSplitHandle):
content = self.handle_pdf_content(file, pdf_document)
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
except BaseException as e:
maxkb_logger.error(f"File: {file.name}, error: {e}, {traceback.format_exc()}")
return {
@ -131,7 +131,7 @@ class PdfSplitHandle(BaseSplitHandle):
return content
@staticmethod
def handle_toc(doc, limit):
def handle_toc(doc, limit, overlap: int = 0):
# 找到目录
toc = doc.get_toc()
if toc is None or len(toc) == 0:
@ -183,7 +183,7 @@ class PdfSplitHandle(BaseSplitHandle):
real_chapter_title = chapter_title[:256]
# 限制章节内容长度
if 0 < limit < len(chapter_text):
split_text = smart_split_paragraph(chapter_text, limit)
split_text = smart_split_paragraph(chapter_text, limit, overlap)
for text in split_text:
chapters.append({"title": real_chapter_title, "content": text})
else:
@ -193,7 +193,7 @@ class PdfSplitHandle(BaseSplitHandle):
return chapters
@staticmethod
def handle_links(doc, pattern_list, with_filter, limit):
def handle_links(doc, pattern_list, with_filter, limit, overlap: int = 0):
# 检查文档是否包含内部链接
if not check_links_in_pdf(doc):
return
@ -262,7 +262,7 @@ class PdfSplitHandle(BaseSplitHandle):
# 限制章节内容长度
if 0 < limit < len(chapter_text):
split_text = smart_split_paragraph(chapter_text, limit)
split_text = smart_split_paragraph(chapter_text, limit, overlap)
for text in split_text:
chapters.append({"title": link_title, "content": text})
else:
@ -285,9 +285,9 @@ class PdfSplitHandle(BaseSplitHandle):
except BaseException as e:
maxkb_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
# 插入目录前的部分
page_content = re.sub(r'(?<!。)\n+', '', page_content)
page_content = re.sub(r'(?<!.)\n+', '', page_content)

View File

@ -39,16 +39,16 @@ class TextSplitHandle(BaseSplitHandle):
return True
return False
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
buffer = get_buffer(file)
if type(limit) is str:
limit = int(limit)
if type(with_filter) is str:
with_filter = with_filter.lower() == 'true'
if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit)
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
try:
content = buffer.decode(detect(buffer)['encoding'])
except BaseException as e:

View File

@ -13,6 +13,7 @@ import xlrd
from common.handle.base_split_handle import BaseSplitHandle
from common.utils.logger import maxkb_logger
from common.utils.table_split import split_table_rows
def post_cell(cell_value):
@ -24,7 +25,7 @@ def row_to_md(row):
[post_cell(str(cell)) if cell is not None else '' for cell in row]) + ' |\n'
def handle_sheet(file_name, sheet, limit: int):
def handle_sheet(file_name, sheet, limit: int, overlap: int = 0):
rows = iter([sheet.row_values(i) for i in range(sheet.nrows)])
paragraphs = []
result = {'name': file_name, 'content': paragraphs}
@ -37,27 +38,12 @@ def handle_sheet(file_name, sheet, limit: int):
return result
if len(title_row_list) == 0:
return result
result_item_content = ''
for row in rows:
next_md_content = row_to_md(row)
next_md_content_len = len(next_md_content)
result_item_content_len = len(result_item_content)
if len(result_item_content) == 0:
result_item_content += title_md_content
result_item_content += next_md_content
else:
if result_item_content_len + next_md_content_len < limit:
result_item_content += next_md_content
else:
paragraphs.append({'content': result_item_content, 'title': ''})
result_item_content = title_md_content + next_md_content
if len(result_item_content) > 0:
paragraphs.append({'content': result_item_content, 'title': ''})
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in rows), limit, overlap))
return result
class XlsSplitHandle(BaseSplitHandle):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
buffer = get_buffer(file)
try:
if type(limit) is str:
@ -67,8 +53,8 @@ class XlsSplitHandle(BaseSplitHandle):
worksheets_size = len(worksheets)
return [row for row in
[handle_sheet(file.name,
sheet, limit) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet(
sheet.name, sheet, limit) for sheet
sheet, limit, overlap) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet(
sheet.name, sheet, limit, overlap) for sheet
in worksheets] if row is not None]
except Exception as e:
maxkb_logger.error(f"Error processing XLS file {file.name}: {e}, {traceback.format_exc()}")

View File

@ -15,6 +15,7 @@ import openpyxl
from common.handle.base_split_handle import BaseSplitHandle
from common.handle.impl.common_handle import xlsx_embed_cells_images
from common.utils.logger import maxkb_logger
from common.utils.table_split import split_table_rows
def post_cell(image_dict, cell_value):
@ -30,7 +31,7 @@ def row_to_md(row, image_dict):
in row]) + ' |\n'
def handle_sheet(file_name, sheet, image_dict, limit: int):
def handle_sheet(file_name, sheet, image_dict, limit: int, overlap: int = 0):
rows = sheet.rows
paragraphs = []
result = {'name': file_name, 'content': paragraphs}
@ -43,32 +44,17 @@ def handle_sheet(file_name, sheet, image_dict, limit: int):
return result
if len(title_row_list) == 0:
return result
result_item_content = ''
for row in rows:
next_md_content = row_to_md(row, image_dict)
next_md_content_len = len(next_md_content)
result_item_content_len = len(result_item_content)
if len(result_item_content) == 0:
result_item_content += title_md_content
result_item_content += next_md_content
else:
if result_item_content_len + next_md_content_len < limit:
result_item_content += next_md_content
else:
paragraphs.append({'content': result_item_content, 'title': ''})
result_item_content = title_md_content + next_md_content
if len(result_item_content) > 0:
paragraphs.append({'content': result_item_content, 'title': ''})
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row, image_dict) for row in rows), limit, overlap))
return result
class XlsxSplitHandle(BaseSplitHandle):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
buffer = get_buffer(file)
try:
if type(limit) is str:
limit = int(limit)
workbook = openpyxl.load_workbook(io.BytesIO(buffer))
workbook = openpyxl.load_workbook(io.BytesIO(buffer), data_only=True)
try:
image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer))
save_image([item for item in image_dict.values()])
@ -80,8 +66,8 @@ class XlsxSplitHandle(BaseSplitHandle):
[handle_sheet(file.name,
sheet,
image_dict,
limit) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet(
sheet.title, sheet, image_dict, limit) for sheet
limit, overlap) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet(
sheet.title, sheet, image_dict, limit, overlap) for sheet
in worksheets] if row is not None]
except Exception as e:
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")

View File

@ -50,11 +50,11 @@ split_handles = [
]
def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image):
def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image, overlap: int = 0):
get_buffer = FileBufferHandle().get_buffer
for split_handle in split_handles:
if split_handle.support(file, get_buffer):
return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image)
return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image, overlap)
raise Exception(_('Unsupported file format'))
@ -116,7 +116,7 @@ def filter_image_file(result_list: list, image_list):
class ZipSplitHandle(BaseSplitHandle):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
if type(limit) is str:
limit = int(limit)
if type(with_filter) is str:
@ -137,7 +137,7 @@ class ZipSplitHandle(BaseSplitHandle):
try:
# 处理一下文件名
f.name = get_file_name(f.name)
value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image)
value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image, overlap)
if isinstance(value, list):
result = [*result, *value]
else:

View File

@ -26,7 +26,8 @@ class GunicornLocalModelService(BaseService):
print("\n- Start Gunicorn Local Model WSGI HTTP Server")
log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
worker = CONFIG.get("LOCAL_MODEL_HOST_WORKER", 1)
# Weights and the load queue have one owner; inference remains multithreaded.
worker = 1
max_requests = 10240 if int(worker) > 1 else 0
cmd = [
'gunicorn', 'maxkb.wsgi:application',

View File

@ -0,0 +1,202 @@
"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
import contextlib
import itertools
import queue
import threading
import time
import uuid
from concurrent.futures import Future
def loading_policy(value=None):
value = {} if value is None else value
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
raise ValueError('加载策略无效')
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
'idle_seconds': value.get('idle_seconds', 300)}
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
raise ValueError('加载方式无效')
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
raise ValueError('加载优先级须为0~100的整数')
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
raise ValueError('空闲时间须为30~86400秒')
return result
class ModelRuntime:
def __init__(self, specs, adapter, clock=time.monotonic):
self.specs, self.adapter, self.clock = specs, adapter, clock
self.entries = {}
self.lock = threading.RLock()
self.queue = queue.PriorityQueue()
self.sequence = itertools.count()
self.load_sequence = itertools.count(1)
self.worker = None
def _entry(self, model_id):
model_id = str(model_id)
spec = self.specs(model_id)
spec['policy'] = loading_policy(spec.get('policy'))
entry = self.entries.get(model_id)
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
if entry['leases'] or entry['state'] == 'LOADING':
raise ValueError('模型配置已变更,请等待当前请求结束')
self._discard(entry)
entry = None
if entry is None:
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
'load_order': None}
self.entries[model_id] = entry
previous = entry['spec']['policy']
entry['spec'] = spec
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
entry['generation'] += 1
self._put(entry)
return entry
def _put(self, entry):
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
def start(self):
with self.lock:
if self.worker is None:
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
self.worker.start()
def submit(self, model_id, start=True):
with self.lock:
entry = self._entry(model_id)
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
entry.update(state='UNLOADED', value=None, info={})
if entry['state'] == 'READY':
return entry
if entry['state'] not in ('LOADING', 'QUEUED'):
entry.update(state='QUEUED', error='', future=Future())
entry['generation'] += 1
self._put(entry)
if start:
self.start()
return entry
def preload(self, model_ids):
# Enqueue the complete startup batch before choosing the highest-priority model.
with self.lock:
for model_id in model_ids:
self.submit(model_id, start=False)
self.start()
def _work(self):
while True:
_, order, model_id, generation = self.queue.get()
try:
with self.lock:
entry = self.entries.get(model_id)
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
continue
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
spec = entry['spec']
future = entry['future']
value, info = self.adapter.load(spec)
with self.lock:
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
future.set_result(True)
del value # The loader thread must not keep the last model alive after eviction.
except Exception as exc:
with self.lock:
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
if not entry['future'].done():
entry['future'].set_exception(ValueError(entry['error']))
finally:
self.queue.task_done()
def acquire(self, model_id, remote=False, timeout=600):
with self.lock:
entry = self.submit(model_id)
token = uuid.uuid4().hex
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
future = entry['future'] if entry['state'] != 'READY' else None
try:
if future:
future.result(timeout=timeout)
return token
except Exception:
self.release(model_id, token)
raise
def release(self, model_id, token):
with self.lock:
entry = self.entries.get(str(model_id))
if entry:
entry['leases'].pop(token, None)
entry['last_used'] = self.clock()
@contextlib.contextmanager
def inference(self, model_id):
token = self.acquire(model_id)
try:
with self.lock:
value = self.entries[str(model_id)]['value']
yield value
finally:
self.release(model_id, token)
def _discard(self, entry):
if entry['value'] is not None:
value = entry['value']
self.adapter.unload(entry['spec'], value)
entry['value'] = None
del value
self.adapter.collect()
entry.update(state='UNLOADED', loaded_at=None, info={})
def unload(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
raise ValueError('模型正在使用或加载,请稍后再试')
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
entry['value'] = True
self._discard(entry)
def configure(self, model_id):
with self.lock:
entry = self._entry(model_id)
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
if warm_again:
entry.update(state='UNLOADED', value=None)
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
self.submit(model_id)
def _expire_leases(self, entry):
now = self.clock()
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
def sweep(self):
with self.lock:
for entry in list(self.entries.values()):
self._expire_leases(entry)
policy = entry['spec']['policy']
# Ollama owns its idle timer; do not unload models used by other clients.
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
self.clock() - entry['last_used'] >= policy['idle_seconds']):
self._discard(entry)
def status(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
# Reconcile native residency, including expiration outside this process.
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
info = self.adapter.probe(entry['spec'])
if info:
entry.update(state='READY', info=info, value=True)
elif entry['state'] == 'READY':
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
'active_requests': len(entry['leases']), 'error': entry['error'],
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
'load_order': entry['load_order'], **entry['info']}

View File

@ -0,0 +1,89 @@
"""Authenticated internal model-service RPC; never forwards user-supplied endpoints."""
import contextlib
import asyncio
import hashlib
import hmac
import json
import os
import time
import uuid
from functools import lru_cache
import requests
@lru_cache(maxsize=1)
def _secret():
if os.environ.get('SERVER_NAME') == 'local_model':
from local_model.models.system_setting import SystemSetting
else:
from system_manage.models import SystemSetting
meta = SystemSetting.objects.filter(type=1).values_list('meta', flat=True).first()
if not meta or not meta.get('value'):
raise ValueError('模型服务认证未配置')
return hashlib.sha256(('model-runtime-v1:' + meta['value']).encode()).digest()
def signed_request(payload):
data = {**payload, 'timestamp': time.time(), 'nonce': uuid.uuid4().hex}
body = json.dumps(data, separators=(',', ':'), sort_keys=True).encode()
signature = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
return body, signature
def verify_request(body, signature):
expected = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
if not signature or not hmac.compare_digest(expected, signature):
raise ValueError('模型服务认证失败')
data = json.loads(body)
if abs(time.time() - float(data.get('timestamp', 0))) > 60:
raise ValueError('模型服务请求已过期')
return data
def runtime_request(operation, model_id, **extra):
from maxkb.const import CONFIG
body, signature = signed_request({'operation': operation, 'model_id': str(model_id), **extra})
host = CONFIG.get('LOCAL_MODEL_HOST')
url = f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{host}:{CONFIG.get("LOCAL_MODEL_PORT")}{CONFIG.get_admin_path()}/api/model_runtime'
try:
response = requests.post(url, data=body, headers={'Content-Type': 'application/json',
'X-Model-Runtime-Signature': signature},
timeout=(5, 620 if operation == 'acquire' else 15))
result = response.json()
except (requests.RequestException, ValueError) as exc:
raise ValueError('本地模型服务不可用') from exc
if response.status_code != 200 or result.get('code') != 200:
raise ValueError(result.get('message') or '模型服务请求失败')
return result['data']
@contextlib.contextmanager
def inference_lease(model_id):
if not model_id:
# Credential validation has no saved resource and must keep its original behavior.
yield None
return
data = runtime_request('acquire', model_id)
try:
yield data
finally:
try:
runtime_request('release', model_id, lease=data['lease'])
except ValueError:
pass # Server expires abandoned leases; never hide the inference result.
@contextlib.asynccontextmanager
async def async_inference_lease(model_id):
if not model_id:
yield None
return
data = await asyncio.to_thread(runtime_request, 'acquire', model_id)
try:
yield data
finally:
try:
await asyncio.to_thread(runtime_request, 'release', model_id, lease=data['lease'])
except ValueError:
pass

View File

@ -274,16 +274,22 @@ def post_handler_paragraph(content: str, limit: int):
return reduce(lambda x, y: [*x, *y], map(lambda row: re.findall(pattern, row), result), [])
def smart_split_paragraph(content: str, limit: int):
def smart_split_paragraph(content: str, limit: int, overlap: int = 0):
"""
智能分段:在limit前找到合适的分割点(句号、回车等)
:param content: 需要分段的文本
:param limit: 最大字符限制
:param overlap: 相邻分段的重叠长度
:return: 分段后的文本列表
"""
if len(content) <= limit:
return [content]
if overlap is None or overlap < 0:
overlap = 0
if overlap >= limit:
overlap = max(0, limit // 2)
result = []
start = 0
@ -318,7 +324,9 @@ def smart_split_paragraph(content: str, limit: int):
best_split = end
result.append(content[start:best_split])
start = best_split
# 相邻分段保留 overlap 字符重叠,且保证向前推进
next_start = best_split - overlap if overlap > 0 else best_split
start = next_start if next_start > start else best_split
return [text for text in result if text.strip()]
@ -345,7 +353,7 @@ def filter_special_char(content: str):
class SplitModel:
def __init__(self, content_level_pattern, with_filter=True, limit=100000):
def __init__(self, content_level_pattern, with_filter=True, limit=100000, overlap=0):
self.content_level_pattern = content_level_pattern
self.with_filter = with_filter
if type(limit) is not int:
@ -355,6 +363,16 @@ class SplitModel:
if limit < 50:
limit = 50
self.limit = limit
if type(overlap) is not int:
try:
overlap = int(overlap)
except (TypeError, ValueError):
overlap = 0
if overlap is None or overlap < 0:
overlap = 0
if overlap >= self.limit:
overlap = max(0, self.limit // 2)
self.overlap = overlap
def parse_to_tree(self, text: str, index=0):
"""
@ -365,7 +383,8 @@ class SplitModel:
"""
level_content_list = parse_title_level(text, self.content_level_pattern, index)
if len(level_content_list) == 0:
return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit)]
return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit,
overlap=self.overlap)]
if index == 0 and text.lstrip().index(level_content_list[0]["content"].lstrip()) != 0:
level_content_list.insert(0, to_tree_obj(""))
@ -374,7 +393,7 @@ class SplitModel:
for i in range(len(level_title_content_list)):
start_content: str = level_title_content_list[i].get('content')
if cursor < text.index(start_content, cursor):
for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit):
for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit, overlap=self.overlap):
level_content_list.insert(0, to_tree_obj(row, 'block'))
block, cursor = get_level_block(text, level_title_content_list, i, cursor)

View File

@ -0,0 +1,27 @@
"""按完整表格行分段;表头重复,重叠行不跨表/工作表。"""
def split_table_rows(header, rows, limit, overlap=0):
limit, overlap = int(limit), int(overlap)
if limit <= 0 or not 0 <= overlap < limit:
raise ValueError("Invalid table splitting limits")
chunks, current = [], []
length = len(header)
for row in rows:
if current and length + len(row) > limit:
chunks.append({"content": header + "".join(current), "title": ""})
carry, carried = [], 0
if overlap:
for previous in reversed(current):
carry.insert(0, previous)
carried += len(previous)
if carried >= overlap:
break
# 必须留出至少一条新行的空间,不能只重复旧行。
while carry and len(header) + sum(map(len, carry)) + len(row) > limit:
carry.pop(0)
current = carry
length = len(header) + sum(map(len, current))
current.append(row)
length += len(row)
if current:
chunks.append({"content": header + "".join(current), "title": ""})
return chunks

View File

@ -16,7 +16,7 @@ from folders.api.folder import FolderCreateRequest
from knowledge.models import KnowledgeFolder, Knowledge
from knowledge.serializers.knowledge import KnowledgeSerializer
from knowledge.serializers.knowledge_folder import KnowledgeFolderTreeSerializer
from system_manage.models import WorkspaceUserResourcePermission
from system_manage.models import WorkspaceUserResourcePermission, EffectiveResourcePermission
from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer
from tools.models import ToolFolder, Tool
from tools.serializers.tool import ToolSerializer
@ -101,7 +101,7 @@ def get_max_depth(current_node):
def has_target_permission(workspace_id, source, user_id, target):
return QuerySet(WorkspaceUserResourcePermission).filter(workspace_id=workspace_id, user_id=user_id,
return QuerySet(EffectiveResourcePermission).filter(workspace_id=workspace_id, user_id=user_id,
auth_target_type=source, target=target,
permission_list__contains=['MANAGE']).exists()
@ -245,17 +245,20 @@ class FolderSerializer(serializers.Serializer):
nodes = Folder.objects.filter(id=self.data.get('id')).get_descendants(include_self=True)
for node in nodes:
# 删除相关的资源
source_ids = (Source.objects.filter(folder_id=node.id)
source_query = Source.objects.filter(folder_id=node.id)
if Source is Knowledge:
source_query = source_query.exclude(meta__has_key='_version_snapshot')
source_ids = (source_query
.annotate(id_str=Cast('id', TextField()))
.values_list('id_str', flat=True))
# 检查文件夹是否存在未授权当前用户的资源
auth_list = QuerySet(WorkspaceUserResourcePermission).filter(
auth_list = QuerySet(EffectiveResourcePermission).filter(
Q(workspace_id=self.data.get('workspace_id')) &
Q(user_id=self.data.get('user_id')) &
Q(auth_target_type=self.data.get('source')) &
Q(target__in=source_ids) &
Q(permission_list__overlap=[ResourcePermission.MANAGE, ResourcePermissionRole.ROLE])
).count()
).values('target').distinct().count()
if auth_list != len(source_ids):
raise AppApiException(500, _('This folder contains resources that you dont have permission'))
self.delete_source(node)
@ -263,7 +266,10 @@ class FolderSerializer(serializers.Serializer):
def delete_source(self, node):
Source = get_source_type(self.data.get('source')) # noqa
source_ids = Source.objects.filter(folder_id=node.id).values_list('id', flat=True)
source_query = Source.objects.filter(folder_id=node.id)
if Source is Knowledge:
source_query = source_query.exclude(meta__has_key='_version_snapshot')
source_ids = source_query.values_list('id', flat=True)
source = self.data.get('source')
for source_id in source_ids:
@ -319,7 +325,7 @@ class FolderTreeSerializer(serializers.Serializer):
if name is not None:
base_q &= Q(name__contains=name)
if not workspace_manage:
base_q &= (Q(id__in=WorkspaceUserResourcePermission.objects.filter(user_id=current_user.id,
base_q &= (Q(id__in=EffectiveResourcePermission.objects.filter(user_id=current_user.id,
auth_target_type=self.data.get('source'),
workspace_id=self.data.get(
'workspace_id'),

View File

@ -0,0 +1,49 @@
"""Guard changes that would invalidate linked assistants or version snapshots."""
from django.db.models import OuterRef, Subquery
from application.models import Application, ApplicationKnowledgeMapping, ApplicationVersion
from knowledge.models import KnowledgeVersion
VERSION_REASON = '知识库已有历史版本,暂不支持更换嵌入模型或影响向量结果的参数。'
ASSISTANT_REASON = '关联助手使用了多个知识库,暂不支持更换嵌入模型或影响向量结果的参数。'
def flow_knowledge_ids(value):
"""Find static knowledge references, including nodes nested in loop bodies."""
ids = set()
if isinstance(value, dict):
if value.get('type') == 'search-knowledge-node':
data = (value.get('properties') or {}).get('node_data') or {}
ids.update(str(item) for item in (data.get('knowledge_id_list') or []) if item)
for child in value.values():
if isinstance(child, (dict, list)):
ids.update(flow_knowledge_ids(child))
elif isinstance(value, list):
for child in value:
if isinstance(child, (dict, list)):
ids.update(flow_knowledge_ids(child))
return ids
def embedding_change_reason(knowledge):
if KnowledgeVersion.objects.filter(knowledge_id=knowledge.id).exists():
return VERSION_REASON
target = str(knowledge.id)
mappings = {}
for app_id, knowledge_id in ApplicationKnowledgeMapping.objects.values_list('application_id', 'knowledge_id'):
mappings.setdefault(str(app_id), set()).add(str(knowledge_id))
for app in Application.objects.only('id', 'type', 'work_flow').iterator():
ids = mappings.get(str(app.id), set()) | flow_knowledge_ids(app.work_flow)
if target in ids and len(ids) > 1:
return ASSISTANT_REASON
latest = ApplicationVersion.objects.filter(application_id=OuterRef('application_id')).order_by(
'-create_time', '-id').values('id')[:1]
for version in ApplicationVersion.objects.filter(id=Subquery(latest)).only(
'application_id', 'knowledge_setting', 'work_flow').iterator():
setting = version.knowledge_setting or {}
ids = {str(item) for item in setting.get('_knowledge_ids', mappings.get(str(version.application_id), set()))}
ids |= flow_knowledge_ids(version.work_flow)
if target in ids and len(ids) > 1:
return ASSISTANT_REASON
return ''

View File

@ -0,0 +1,66 @@
"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
import math
import unicodedata
from langchain_core.embeddings import Embeddings
def validate_params(params):
if (not isinstance(params, dict) or len(params) > 52 or
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
raise ValueError('可选参数最多50项')
result, names = {}, set()
for key, value in params.items():
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
raise ValueError('参数名称须为1~64个字符')
key = key.strip()
normalized = unicodedata.normalize('NFKC', key).casefold()
if normalized in names:
raise ValueError('参数名称不能重复')
names.add(normalized)
if value is not None and (not isinstance(value, (str, int, float, bool)) or
isinstance(value, str) and len(value) > 2000 or
isinstance(value, float) and not math.isfinite(value)):
raise ValueError('参数值须为2000字以内的文本或数值')
result[key] = value
batch = result.get('batch_size')
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
raise ValueError('批处理数量须为1~256的整数')
if result.get('normalize') is not None and type(result['normalize']) is not bool:
raise ValueError('向量归一化参数无效')
for key in ('query_prefix', 'document_prefix'):
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
raise ValueError('向量指令须为2000字以内的文本')
return {key: value for key, value in result.items() if value is not None and
(value != '' or key not in {'query_prefix', 'document_prefix'})}
def knowledge_params(knowledge):
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
params.setdefault('normalize', True)
return {key: value for key, value in params.items()
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
class ConfiguredEmbedding(Embeddings):
def __init__(self, model, params):
self.model = model
self.params = params
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
def _vector(self, vector):
if not self.params.get('normalize', True):
return vector
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
return [float(v) / norm for v in vector] if norm else vector
def embed_query(self, text):
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
def embed_documents(self, texts):
prefix = self.params.get('document_prefix', '')
result = []
for start in range(0, len(texts), self.batch_size):
result.extend(self._vector(v) for v in self.model.embed_documents(
[prefix + text for text in texts[start:start + self.batch_size]]))
return result

View File

@ -0,0 +1,39 @@
# Generated for independent knowledge index rebuild task
import uuid_utils.compat
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0003_tag_documenttag'),
('users', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='KnowledgeRebuildTask',
fields=[
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')),
('status', models.CharField(choices=[('PENDING', '待执行'), ('RUNNING', '执行中'), ('SUCCESS', '成功'), ('PARTIAL', '部分成功'), ('FAILED', '失败')], db_index=True, default='PENDING', max_length=20, verbose_name='任务状态')),
('config_snapshot', models.JSONField(default=dict, verbose_name='配置快照')),
('document_total', models.IntegerField(default=0, verbose_name='文档总数')),
('document_success', models.IntegerField(default=0, verbose_name='成功文档数')),
('document_failed', models.IntegerField(default=0, verbose_name='失败文档数')),
('error_message', models.TextField(blank=True, default='', verbose_name='失败原因')),
('started_at', models.DateTimeField(blank=True, null=True, verbose_name='开始时间')),
('finished_at', models.DateTimeField(blank=True, null=True, verbose_name='结束时间')),
('celery_task_id', models.CharField(blank=True, default='', max_length=128, verbose_name='Celery任务id')),
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='rebuild_tasks', to='knowledge.knowledge', verbose_name='知识库')),
('operator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='操作人')),
],
options={
'db_table': 'knowledge_rebuild_task',
'indexes': [models.Index(fields=['knowledge', 'status'], name='knowledge_r_knowled_0a6e2f_idx')],
},
),
]

View File

@ -0,0 +1,17 @@
# Generated for rebuild task attempt polling
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0004_knowledge_rebuild_task'),
]
operations = [
migrations.AddField(
model_name='knowledgerebuildtask',
name='attempt',
field=models.IntegerField(default=0, verbose_name='汇总轮询次数'),
),
]

View File

@ -0,0 +1,82 @@
# Generated by Django 5.2.8 on 2026-10-27 03:14
import django.db.models.deletion
import uuid_utils.compat
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('knowledge', '0005_rebuild_task_attempt'),
('users', '0002_user_activation_and_role_assignment'),
]
operations = [
migrations.AddField(
model_name='knowledge',
name='version_sequence',
field=models.PositiveIntegerField(default=0, editable=False),
),
migrations.CreateModel(
name='KnowledgeVersion',
fields=[
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False)),
('number', models.PositiveIntegerField()),
('name', models.CharField(max_length=128)),
('description', models.CharField(blank=True, default='', max_length=512)),
('creator_name', models.CharField(default='', max_length=128)),
('document_count', models.PositiveIntegerField(default=0)),
('paragraph_count', models.PositiveIntegerField(default=0)),
('vector_count', models.PositiveIntegerField(default=0)),
('search_ready', models.BooleanField(default=False)),
('content_hash', models.CharField(max_length=64)),
('config', models.JSONField(default=dict)),
],
options={
'db_table': 'knowledge_version',
'ordering': ['-number'],
},
),
migrations.CreateModel(
name='KnowledgeVersionLease',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('chat_id', models.CharField(max_length=128)),
('expires_at', models.DateTimeField(db_index=True)),
],
options={
'db_table': 'knowledge_version_lease',
},
),
migrations.AddField(
model_name='knowledgeversion',
name='creator',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='users.user'),
),
migrations.AddField(
model_name='knowledgeversion',
name='knowledge',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='versions', to='knowledge.knowledge'),
),
migrations.AddField(
model_name='knowledgeversion',
name='snapshot',
field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='knowledge.knowledge'),
),
migrations.AddField(
model_name='knowledgeversionlease',
name='version',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='leases', to='knowledge.knowledgeversion'),
),
migrations.AddConstraint(
model_name='knowledgeversion',
constraint=models.UniqueConstraint(fields=('knowledge', 'number'), name='knowledge_version_number_unique'),
),
migrations.AddConstraint(
model_name='knowledgeversionlease',
constraint=models.UniqueConstraint(fields=('version', 'chat_id'), name='knowledge_version_chat_unique'),
),
]

View File

@ -1 +1,2 @@
from .knowledge import *
from .version import KnowledgeVersion, KnowledgeVersionLease

View File

@ -56,6 +56,46 @@ class KnowledgeScope(models.TextChoices):
WORKSPACE = "WORKSPACE", "工作空间可用"
class RebuildStatus(models.TextChoices):
PENDING = "PENDING", "待执行"
RUNNING = "RUNNING", "执行中"
SUCCESS = "SUCCESS", "成功"
PARTIAL = "PARTIAL", "部分成功"
FAILED = "FAILED", "失败"
class KnowledgeRebuildTask(AppModelMixin):
"""知识库独立索引重建任务。
与“更换向量模型自动触发重向量化”分开:本表记录管理员在配置不变时
主动发起的重建请求、执行状态、文档统计和失败原因。
"""
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.CASCADE, db_constraint=False,
related_name="rebuild_tasks", verbose_name="知识库")
workspace_id = models.CharField(max_length=64, verbose_name="工作空间id", default="default", db_index=True)
status = models.CharField(max_length=20, choices=RebuildStatus.choices, default=RebuildStatus.PENDING,
db_index=True, verbose_name="任务状态")
config_snapshot = models.JSONField(verbose_name="配置快照", default=dict)
document_total = models.IntegerField(verbose_name="文档总数", default=0)
document_success = models.IntegerField(verbose_name="成功文档数", default=0)
document_failed = models.IntegerField(verbose_name="失败文档数", default=0)
error_message = models.TextField(verbose_name="失败原因", default="", blank=True)
operator = models.ForeignKey(User, on_delete=models.SET_NULL, db_constraint=False, null=True, blank=True,
related_name="+", verbose_name="操作人")
started_at = models.DateTimeField(verbose_name="开始时间", null=True, blank=True)
finished_at = models.DateTimeField(verbose_name="结束时间", null=True, blank=True)
celery_task_id = models.CharField(max_length=128, verbose_name="Celery任务id", default="", blank=True)
attempt = models.IntegerField(verbose_name="汇总轮询次数", default=0)
class Meta:
db_table = "knowledge_rebuild_task"
indexes = [
models.Index(fields=["knowledge", "status"], name='knowledge_r_knowled_0a6e2f_idx'),
]
class HitHandlingMethod(models.TextChoices):
optimization = 'optimization', '模型优化'
directly_return = 'directly_return', '直接返回'
@ -130,6 +170,7 @@ class Knowledge(AppModelMixin):
file_size_limit = models.IntegerField(verbose_name="文件大小限制", default=100)
file_count_limit = models.IntegerField(verbose_name="文件数量限制", default=50)
meta = models.JSONField(verbose_name="元数据", default=dict)
version_sequence = models.PositiveIntegerField(default=0, editable=False)
class Meta:
db_table = "knowledge"

View File

@ -0,0 +1,36 @@
"""Immutable knowledge releases and short-lived conversation references."""
import uuid_utils.compat as uuid
from django.db import models
from common.mixins.app_model_mixin import AppModelMixin
class KnowledgeVersion(AppModelMixin):
id = models.UUIDField(primary_key=True, default=uuid.uuid7, editable=False)
knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.PROTECT, related_name='versions')
snapshot = models.OneToOneField('knowledge.Knowledge', on_delete=models.PROTECT, related_name='+')
number = models.PositiveIntegerField()
name = models.CharField(max_length=128)
description = models.CharField(max_length=512, default='', blank=True)
creator = models.ForeignKey('users.User', null=True, on_delete=models.SET_NULL)
creator_name = models.CharField(max_length=128, default='')
document_count = models.PositiveIntegerField(default=0)
paragraph_count = models.PositiveIntegerField(default=0)
vector_count = models.PositiveIntegerField(default=0)
search_ready = models.BooleanField(default=False)
content_hash = models.CharField(max_length=64)
config = models.JSONField(default=dict)
class Meta:
db_table = 'knowledge_version'
ordering = ['-number']
constraints = [models.UniqueConstraint(fields=['knowledge', 'number'], name='knowledge_version_number_unique')]
class KnowledgeVersionLease(models.Model):
version = models.ForeignKey(KnowledgeVersion, on_delete=models.CASCADE, related_name='leases')
chat_id = models.CharField(max_length=128)
expires_at = models.DateTimeField(db_index=True)
class Meta:
db_table = 'knowledge_version_lease'
constraints = [models.UniqueConstraint(fields=['version', 'chat_id'], name='knowledge_version_chat_unique')]

View File

@ -114,40 +114,31 @@ class ProblemParagraphManage:
def get_embedding_model_by_knowledge_id_list(knowledge_id_list: List):
knowledge_list = QuerySet(Knowledge).filter(id__in=knowledge_id_list)
if len(set([knowledge.embedding_model_id for knowledge in knowledge_list])) > 1:
raise Exception(_('The knowledge base is inconsistent with the vector model'))
if len(knowledge_list) == 0:
knowledge_list = list(QuerySet(Knowledge).select_related('embedding_model').filter(id__in=knowledge_id_list))
if not knowledge_list:
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
if knowledge_list[0].embedding_model_id is None or knowledge_list[0].embedding_model is None:
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
default_params = get_model_default_params(knowledge_list[0].embedding_model)
return ModelManage.get_model(
str(knowledge_list[0].embedding_model_id),
lambda _id: get_model(knowledge_list[0].embedding_model, **{**default_params})
)
from knowledge.embedding_params import knowledge_params
configs = [(str(k.embedding_model_id), {key: value for key, value in knowledge_params(k).items()
if key != 'batch_size'}) for k in knowledge_list]
if any(config != configs[0] for config in configs[1:]):
raise Exception('知识库的向量模型或嵌入参数不一致')
return get_embedding_model_by_knowledge(knowledge_list[0])
def get_embedding_model_by_knowledge_id(knowledge_id: str):
knowledge = QuerySet(Knowledge).select_related('embedding_model').filter(id=knowledge_id).first()
if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None:
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
default_params = get_model_default_params(knowledge.embedding_model)
return ModelManage.get_model(str(knowledge.embedding_model_id),
lambda _id: get_model(knowledge.embedding_model, **{**default_params}))
return get_embedding_model_by_knowledge(knowledge)
def get_embedding_model_by_knowledge(knowledge):
if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None:
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
from knowledge.embedding_params import ConfiguredEmbedding, knowledge_params
default_params = get_model_default_params(knowledge.embedding_model)
return ModelManage.get_model(str(knowledge.embedding_model_id),
lambda _id: get_model(knowledge.embedding_model, **{**default_params}))
model = ModelManage.get_model(str(knowledge.embedding_model_id),
lambda _id: get_model(knowledge.embedding_model, **default_params))
params = knowledge_params(knowledge)
return ConfiguredEmbedding(model, params) if params else model
def get_embedding_model_id_by_knowledge_id(knowledge_id):

View File

@ -162,7 +162,8 @@ class DocumentEditInstanceSerializer(serializers.Serializer):
class DocumentSplitRequest(serializers.Serializer):
file = serializers.ListField(required=True, label=_('file list'))
limit = serializers.IntegerField(required=False, label=_('limit'))
limit = serializers.IntegerField(required=False, min_value=50, max_value=100000, label=_('limit'))
overlap = serializers.IntegerField(required=False, min_value=0, label=_('chunk overlap'))
patterns = serializers.ListField(
required=False,
child=serializers.CharField(required=True, label=_('patterns')),
@ -338,7 +339,8 @@ class DocumentSerializers(serializers.Serializer):
else:
document_list.update(knowledge_id=target_knowledge_id)
model_id = None
if knowledge.embedding_model_id != target_knowledge.embedding_model_id:
if (knowledge.embedding_model_id != target_knowledge.embedding_model_id or
(knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})):
model_id = get_embedding_model_id_by_knowledge_id(target_knowledge_id)
pid_list = [paragraph.id for paragraph in paragraph_list]
@ -1047,16 +1049,38 @@ class DocumentSerializers(serializers.Serializer):
def parse(self, instance):
self.is_valid(instance=instance, raise_exception=True)
DocumentSplitRequest(data=instance).is_valid(raise_exception=True)
split_request = DocumentSplitRequest(data=instance)
split_request.is_valid(raise_exception=True)
instance = split_request.validated_data
# 分段长度/重叠未显式指定时,回退到知识库分段规则(meta.segmentation)
knowledge = QuerySet(Knowledge).filter(id=self.data.get('knowledge_id')).first()
segmentation = (knowledge.meta or {}).get('segmentation') if knowledge is not None else None
segmentation = segmentation or {}
default_limit = segmentation.get('chunk_size') or 500
default_overlap = segmentation.get('chunk_overlap') or 0
limit = instance.get("limit", None) or default_limit
overlap = instance.get("overlap", None)
overlap = default_overlap if overlap is None else overlap
if not 0 <= overlap < limit:
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
file_list = instance.get("file")
from system_manage.access_control import enforce
from system_manage.parsers import extension, MINERU_INPUTS
request = self.context.get('request')
enforce(request, 'knowledge', self.data.get('knowledge_id'))
parser_id = (knowledge.meta or {}).get('parser_id') if knowledge else None
if parser_id and any(extension(file.name) in MINERU_INPUTS for file in file_list):
enforce(request, 'parser', parser_id)
return reduce(
lambda x, y: [*x, *y],
[self.file_to_paragraph(
f,
instance.get("patterns", None),
instance.get("with_filter", None),
instance.get("limit", 4096)
limit,
overlap
) for f in file_list],
[]
)
@ -1095,7 +1119,7 @@ class DocumentSerializers(serializers.Serializer):
item['source_file_id'] = file_id
return result_list
def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int):
def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int, overlap: int = 0):
# 保存源文件
file_id = uuid.uuid7()
raw_file = File(
@ -1108,12 +1132,22 @@ class DocumentSerializers(serializers.Serializer):
raw_file.save(file.read())
file.seek(0)
from system_manage.ai_hub_knowledge import split_with_parser
parsed = split_with_parser(self.data.get('workspace_id'), self.data.get('knowledge_id'),
self.context.get('user'), file, pattern_list, with_filter, limit, overlap)
if parsed is not None:
result, provenance = parsed
raw_file.meta = {**(raw_file.meta or {}), 'parser': provenance}
QuerySet(File).filter(id=file_id).update(meta=raw_file.meta)
return self.normalize_split_result(result, file, default_split_handle, file_id)
get_buffer = FileBufferHandle().get_buffer
for split_handle in split_handles:
if split_handle.support(file, get_buffer):
result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image)
result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image,
overlap)
return self.normalize_split_result(result, file, split_handle, file_id)
result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image)
result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image,
overlap)
return self.normalize_split_result(result, file, default_split_handle, file_id)
class SplitPattern(serializers.Serializer):
@ -1706,7 +1740,7 @@ class DocumentSerializers(serializers.Serializer):
data={'workspace_id': self.data.get('workspace_id'), 'knowledge_id': self.data.get('knowledge_id')}
)
split_serializer.is_valid(instance={'file': [file]}, raise_exception=True)
parsed_documents = split_serializer.file_to_paragraph(file, None, None, 4096)
parsed_documents = split_serializer.file_to_paragraph(file, None, None, 500)
paragraphs = []
for parsed_document in parsed_documents:
paragraphs.extend(parsed_document.get('content', []))

View File

@ -70,6 +70,7 @@ class KnowledgeWebCreateRequest(serializers.Serializer):
class KnowledgeEditRequest(serializers.Serializer):
embedding_params = serializers.DictField(required=False)
name = serializers.CharField(required=False, max_length=64, min_length=1, label=_('knowledge name'))
desc = serializers.CharField(required=False, max_length=256, min_length=1, label=_('knowledge description'))
meta = serializers.DictField(required=False)
@ -166,7 +167,8 @@ class KnowledgeSerializer(serializers.Serializer):
query_set_dict['knowledge_custom_sql'] = QuerySet(model=get_dynamics_model({
'knowledge.workspace_id': models.CharField(),
})).filter(**{'knowledge.workspace_id': workspace_id})
'knowledge.meta': models.JSONField(),
})).filter(**{'knowledge.workspace_id': workspace_id}).exclude(**{'knowledge.meta__has_key': '_version_snapshot'})
# query_set_dict['folder_query_set'] = folder_query_set
if not workspace_manage:
query_set_dict['workspace_user_resource_permission_query_set'] = QuerySet(
@ -344,9 +346,12 @@ class KnowledgeSerializer(serializers.Serializer):
)
)
), with_search_one=True)
from knowledge.embedding_guard import embedding_change_reason
return {
**knowledge_dict,
'meta': json.loads(knowledge_dict.get('meta', '{}')),
'embedding_change_block_reason': embedding_change_reason(
QuerySet(Knowledge).get(id=self.data.get('knowledge_id'))),
'application_id_list': list(filter(
lambda application_id: all_application_list.__contains__(application_id),
[
@ -361,7 +366,7 @@ class KnowledgeSerializer(serializers.Serializer):
@transaction.atomic
def edit(self, instance: Dict, select_one=True):
self.is_valid()
knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id"))
knowledge = QuerySet(Knowledge).select_for_update().get(id=self.data.get("knowledge_id"))
if QuerySet(Knowledge).filter(
workspace_id=knowledge.workspace_id,
name=instance.get('name'),
@ -369,6 +374,23 @@ class KnowledgeSerializer(serializers.Serializer):
).exclude(id=knowledge.id).exists():
raise AppApiException(500, _('Knowledge base name duplicate!'))
KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge)
from knowledge.embedding_params import knowledge_params, validate_params
from knowledge.embedding_guard import embedding_change_reason
from types import SimpleNamespace
current_params = knowledge_params(knowledge)
try:
params = validate_params(instance['embedding_params']) if 'embedding_params' in instance else None
except ValueError as exc:
raise AppApiException(500, str(exc))
new_params = knowledge_params(SimpleNamespace(meta={'embedding_params': params})) if params is not None else current_params
model_changed = ('embedding_model_id' in instance and
str(instance['embedding_model_id'] or '') != str(knowledge.embedding_model_id or ''))
vector_params_changed = ({k: v for k, v in current_params.items() if k != 'batch_size'} !=
{k: v for k, v in new_params.items() if k != 'batch_size'})
if model_changed or vector_params_changed:
reason = embedding_change_reason(knowledge)
if reason:
raise AppApiException(500, reason)
if 'embedding_model_id' in instance:
knowledge.embedding_model_id = instance.get('embedding_model_id')
if "name" in instance:
@ -376,7 +398,15 @@ class KnowledgeSerializer(serializers.Serializer):
if 'desc' in instance:
knowledge.desc = instance.get("desc")
if 'meta' in instance:
knowledge.meta = instance.get('meta')
incoming_meta = dict(instance.get('meta') or {})
# Independently saved metadata/rules must survive stale general settings forms.
for managed_key in ('business_metadata', 'segmentation', 'parser_id', 'embedding_params'):
incoming_meta.pop(managed_key, None)
if managed_key in (knowledge.meta or {}):
incoming_meta[managed_key] = knowledge.meta[managed_key]
knowledge.meta = incoming_meta
if 'embedding_params' in instance:
knowledge.meta = {**(knowledge.meta or {}), 'embedding_params': params}
if 'folder_id' in instance:
knowledge.folder_id = instance.get('folder_id')
if 'file_size_limit' in instance:
@ -384,6 +414,10 @@ class KnowledgeSerializer(serializers.Serializer):
if 'file_count_limit' in instance:
knowledge.file_count_limit = instance.get('file_count_limit')
if 'user_id' in instance:
if str(instance.get('user_id')) != str(knowledge.user_id):
from knowledge.serializers.knowledge_ext import _creator_candidates
if not _creator_candidates(knowledge.workspace_id).filter(id=instance.get('user_id')).exists():
raise AppApiException(500, '创建者须为已激活且启用的成员')
knowledge.user_id = instance.get('user_id')
if 'create_user' in instance:
knowledge.create_user = instance.get('create_user')
@ -422,6 +456,9 @@ class KnowledgeSerializer(serializers.Serializer):
def delete(self):
self.is_valid()
knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id"))
from knowledge.models import KnowledgeVersion
if KnowledgeVersion.objects.filter(knowledge=knowledge).exists():
raise AppApiException(500, '请先删除该知识库的历史版本')
QuerySet(Document).filter(knowledge=knowledge).delete()
QuerySet(ProblemParagraphMapping).filter(knowledge=knowledge).delete()
QuerySet(Paragraph).filter(knowledge=knowledge).delete()

View File

@ -0,0 +1,215 @@
# coding=utf-8
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
from collections.abc import Mapping
import re
from django.db import transaction
from django.db.models import QuerySet
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import Knowledge
METADATA_FIELDS = [
{"field": "business_domain", "label": "业务领域"},
{"field": "data_source", "label": "数据来源"},
{"field": "owner_dept", "label": "责任部门"},
{"field": "business_version", "label": "业务版本"},
]
METADATA_MAX_LEN = 64
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
query = QuerySet(Knowledge)
if lock:
query = query.select_for_update()
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
return knowledge
def _creator_candidates(workspace_id):
from users.models import User, UserActivationApplication
from common.database_model_manage.database_model_manage import DatabaseModelManage
from django.db.models import Q
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
query = User.objects.filter(is_active=True).exclude(id__in=pending)
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
if membership is not None:
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
return query
def _metadata_fields(business):
# An explicit empty list means all custom fields were removed; do not revive old fields.
if isinstance(business.get('custom_fields'), list):
return business['custom_fields']
return [{'name': item['label'], 'value': business[item['field']]}
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
def _validate_custom_fields(items):
import unicodedata
if not isinstance(items, list) or len(items) > 50:
raise AppApiException(500, '元数据最多50项')
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
result, names = [], set()
for item in items:
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
raise AppApiException(500, '元数据需包含名称和值')
name, value = item['name'], item['value']
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
raise AppApiException(500, '元数据名称须为1~64个字符')
name = name.strip()
normalized = unicodedata.normalize('NFKC', name).casefold()
if normalized in reserved:
raise AppApiException(500, '不能使用固定属性名称')
if normalized in names:
raise AppApiException(500, '元数据名称不能重复')
if not isinstance(value, str) or len(value) > 2000:
raise AppApiException(500, '元数据值须为2000字以内的文本')
names.add(normalized)
result.append({'name': name, 'value': value.strip()})
return result
class KnowledgeMetadataSerializer(serializers.Serializer):
"""Fixed system attributes plus freely named business metadata; save atomically."""
@staticmethod
def one(workspace_id: str, knowledge_id: str):
from users.models import User
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = (knowledge.meta or {}).get('business_metadata') or {}
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
for u in _creator_candidates(workspace_id).order_by('nick_name')]
if creator and not any(u['id'] == str(creator.id) for u in candidates):
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
return {
'knowledge_id': str(knowledge.id),
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
'creator_name': (creator.nick_name or creator.username) if creator else '',
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
'creator_options': candidates,
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
}
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, '元数据必须为属性对象')
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
if set(data) - (set(legacy) | {'fields', 'user_id'}):
raise AppApiException(500, '包含不支持的元数据属性')
if 'fields' in data and set(data) & set(legacy):
raise AppApiException(500, '不能混用新旧元数据格式')
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
values = {}
for key, label in legacy.items():
if key not in data:
continue
value = '' if data[key] is None else data[key]
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
raise AppApiException(500, f'{label}须为64字以内的文本')
values[key] = value.strip()
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
try:
creator_id = serializers.UUIDField().run_validation(data['user_id'])
except serializers.ValidationError:
raise AppApiException(500, '请选择有效创建者')
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
raise AppApiException(500, '创建者须为已激活且启用的成员')
knowledge.user_id = creator_id
meta = dict(knowledge.meta or {})
business = dict(meta.get('business_metadata') or {})
if fields is not None:
business['custom_fields'] = fields
elif 'custom_fields' in business:
current = list(_metadata_fields(business))
for key, value in values.items():
existing = next((item for item in current if item['name'] == legacy[key]), None)
if existing is not None:
existing['value'] = value
elif value:
current.append({'name': legacy[key], 'value': value})
business['custom_fields'] = _validate_custom_fields(current)
business.update(values)
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
updated_at=_fmt(timezone.now()))
meta['business_metadata'] = business
knowledge.meta = meta
knowledge.save(update_fields=['meta', 'user', 'update_time'])
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
class KnowledgeSegmentationSerializer(serializers.Serializer):
"""知识库分段规则:查看 / 修改。
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
保存后新建文档上传时默认使用该规则,上传时可覆盖。
"""
DEFAULTS = {"chunk_size": 500, "chunk_overlap": 0}
@staticmethod
def one(workspace_id: str, knowledge_id: str):
knowledge = _get_knowledge(workspace_id, knowledge_id)
data = dict((knowledge.meta or {}).get('segmentation') or {})
result = {
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
}
result.update({
"updated_by": data.get("updated_by_name") or "",
"updated_at": data.get("updated_at") or "",
"knowledge_id": str(knowledge.id),
})
return result
@staticmethod
@transaction.atomic
def update(workspace_id: str, knowledge_id: str, user, data: dict):
if not isinstance(data, Mapping):
raise AppApiException(500, "分段规则必须为属性对象")
def integer(value):
if isinstance(value, bool) or not isinstance(value, (int, str)):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
raise AppApiException(500, "分段长度/重叠长度必须为整数")
return int(value)
chunk_size = integer(data.get("chunk_size"))
chunk_overlap = integer(data.get("chunk_overlap", 0))
if chunk_size < 50 or chunk_size > 100000:
raise AppApiException(500, "分段长度取值范围为 50~100000")
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
meta = dict(knowledge.meta or {})
meta['segmentation'] = {
"chunk_size": chunk_size,
"chunk_overlap": chunk_overlap,
"updated_by": str(user.id),
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
"updated_at": _fmt(timezone.now()),
}
knowledge.meta = meta
knowledge.save(update_fields=["meta", "update_time"])
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)

View File

@ -597,7 +597,8 @@ class ParagraphSerializers(serializers.Serializer):
target_knowledge = QuerySet(Knowledge).filter(id=target_knowledge_id).first()
knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first()
embedding_model_id = None
if target_knowledge.embedding_model_id != knowledge.embedding_model_id:
if (target_knowledge.embedding_model_id != knowledge.embedding_model_id or
(knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})):
embedding_model_id = str(target_knowledge.embedding_model_id)
pid_list = [paragraph.id for paragraph in paragraph_list]
# 修改段落信息

View File

@ -0,0 +1,129 @@
# coding=utf-8
"""知识库独立索引重建任务序列化器。"""
from django.db import transaction
from django.utils import timezone
from rest_framework import serializers
from common.exception.app_exception import AppApiException
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
from knowledge.task.rebuild import rebuild_knowledge_index
from models_provider.models import Model
def _fmt(value):
if not value:
return ""
try:
value = timezone.localtime(value)
except Exception:
pass
return value.strftime("%Y-%m-%d %H:%M:%S")
def serialize_task(task: KnowledgeRebuildTask):
total = task.document_total or 0
success = task.document_success or 0
failed = task.document_failed or 0
processed = success + failed
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
return {
"id": str(task.id),
"knowledge_id": str(task.knowledge_id),
"workspace_id": task.workspace_id,
"status": task.status,
"status_label": status_label,
"config_snapshot": task.config_snapshot or {},
"document_total": total,
"document_success": success,
"document_failed": failed,
"progress": progress,
"error_message": task.error_message or "",
"created_at": _fmt(task.create_time),
"started_at": _fmt(task.started_at),
"finished_at": _fmt(task.finished_at),
}
class KnowledgeRebuildSerializer(serializers.Serializer):
@staticmethod
def create(workspace_id: str, knowledge_id: str, user_id: str):
# 锁住知识库行,串行化同一知识库的检查和任务创建。
with transaction.atomic():
knowledge = Knowledge.objects.select_for_update().filter(
id=knowledge_id, workspace_id=workspace_id
).first()
if knowledge is None:
raise AppApiException(500, "知识库不存在")
if KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id,
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).exists():
raise AppApiException(500, "该知识库已有重建任务在执行")
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
if model is None:
raise AppApiException(500, "知识库未配置可用的向量模型")
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge_id).values_list("id", flat=True)]
if not document_ids:
raise AppApiException(500, "知识库没有可重建的文档")
snapshot = {
"embedding_model_id": str(model.id),
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
"embedding_model_name": model.name,
"embedding_model": model.model_name,
"knowledge_name": knowledge.name,
"document_ids": document_ids,
"document_total": len(document_ids),
"snapshot_at": _fmt(timezone.now()),
}
task = KnowledgeRebuildTask.objects.create(
knowledge_id=knowledge_id, workspace_id=workspace_id,
status=RebuildStatus.PENDING, config_snapshot=snapshot,
document_total=len(document_ids), operator_id=user_id,
)
def dispatch():
try:
result = rebuild_knowledge_index.delay(str(task.id))
# 不覆盖 worker 已经更新的状态。
KnowledgeRebuildTask.objects.filter(id=task.id).update(
celery_task_id=getattr(result, "id", "") or "")
except Exception as exc:
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
finished_at=timezone.now(), update_time=timezone.now())
transaction.on_commit(dispatch)
task.refresh_from_db()
return serialize_task(task)
@staticmethod
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
query_set = KnowledgeRebuildTask.objects.filter(
knowledge_id=knowledge_id, workspace_id=workspace_id
).order_by("-create_time")
total = query_set.count()
start = max(0, (current_page - 1) * page_size)
records = [serialize_task(item) for item in query_set[start:start + page_size]]
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
@staticmethod
def one(workspace_id: str, knowledge_id: str, task_id: str):
task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if task is None:
raise AppApiException(500, "重建任务不存在")
return serialize_task(task)
@staticmethod
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
old_task = KnowledgeRebuildTask.objects.filter(
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
).first()
if old_task is None:
raise AppApiException(500, "重建任务不存在")
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)

View File

@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
FROM (SELECT knowledge.*
FROM knowledge knowledge ${knowledge_custom_sql}
AND id::text in (select target
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
from effective_resource_permission ${workspace_user_resource_permission_query_set}
and 'VIEW' = any (permission_list))) temp_knowledge
LEFT JOIN (SELECT "count"("id") AS document_count, "sum"("char_length") "char_length", knowledge_id
FROM "document"

View File

@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
FROM (SELECT knowledge.*
FROM knowledge knowledge ${knowledge_custom_sql}
AND "knowledge".id::text in (select target
from workspace_user_resource_permission
from effective_resource_permission
${workspace_user_resource_permission_query_set}
and case
when auth_type = 'ROLE' then
@ -34,8 +34,8 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
from role_permission role_permission
right join user_role_relation user_role_relation
on user_role_relation.role_id=role_permission.role_id
where user_role_relation.user_id=workspace_user_resource_permission.user_id
and user_role_relation.workspace_id=workspace_user_resource_permission.workspace_id)
where user_role_relation.user_id=effective_resource_permission.user_id
and user_role_relation.workspace_id=effective_resource_permission.workspace_id)
else
'VIEW' = any (permission_list)
end

View File

@ -11,7 +11,7 @@ from common.config.embedding_config import ModelManage
from common.event.listener_manage import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
UpdateEmbeddingDocumentIdArgs
from common.utils.logger import maxkb_logger
from knowledge.models import Document, TaskType, State
from knowledge.models import Document, Paragraph, Problem, TaskType, State
from knowledge.serializers.common import drop_knowledge_index
from models_provider.models import Model
from models_provider.tools import get_model, get_model_default_params
@ -35,21 +35,35 @@ def get_embedding_model(model_id, exception_handler=lambda e: maxkb_logger.error
return embedding_model
def scoped_model(model_id, knowledge_id):
from knowledge.serializers.common import get_embedding_model_by_knowledge_id
if knowledge_id is None:
raise ValueError('无法确定向量化任务的知识库')
return get_embedding_model_by_knowledge_id(knowledge_id)
def paragraph_knowledge(ids):
values = list(QuerySet(Paragraph).filter(id__in=ids).values_list('knowledge_id', flat=True).distinct())
if len(values) != 1:
raise ValueError('段落须属于同一知识库')
return values[0]
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id']}, name='celery:embedding_by_paragraph')
def embedding_by_paragraph(paragraph_id, model_id):
embedding_model = get_embedding_model(model_id)
embedding_model = scoped_model(model_id, paragraph_knowledge([paragraph_id]))
ListenerManagement.embedding_by_paragraph(paragraph_id, embedding_model)
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_data_list')
def embedding_by_paragraph_data_list(data_list, paragraph_id_list, model_id):
embedding_model = get_embedding_model(model_id)
embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list))
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model)
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_list')
def embedding_by_paragraph_list(paragraph_id_list, model_id):
embedding_model = get_embedding_model(model_id)
embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list))
ListenerManagement.embedding_by_paragraph_list(paragraph_id_list, embedding_model)
@ -77,7 +91,11 @@ def embedding_by_document(document_id, model_id, state_list=None):
traceback=traceback.format_exc()
))
embedding_model = get_embedding_model(model_id, exception_handler)
try:
embedding_model = scoped_model(model_id, QuerySet(Document).filter(id=document_id).values_list('knowledge_id', flat=True).first())
except Exception as exc:
exception_handler(exc)
raise
#
ListenerManagement.embedding_by_document(document_id, embedding_model, state_list)
@ -130,12 +148,15 @@ def embedding_by_problem(args, model_id):
@param model_id: 模型id
@return:
"""
embedding_model = get_embedding_model(model_id)
embedding_model = scoped_model(model_id, args.get('knowledge_id'))
ListenerManagement.embedding_by_problem(args, embedding_model)
def embedding_by_data_list(args: List, model_id):
embedding_model = get_embedding_model(model_id)
if not args:
return
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
embedding_model = get_embedding_model_by_knowledge_id_list(list({str(row['knowledge_id']) for row in args}))
ListenerManagement.embedding_by_data_list(args, embedding_model)
@ -220,7 +241,7 @@ def update_problem_embedding(problem_id: str, problem_content: str, model_id):
@param model_id:
@return:
"""
model = get_embedding_model(model_id)
model = scoped_model(model_id, QuerySet(Problem).filter(id=problem_id).values_list('knowledge_id', flat=True).first())
ListenerManagement.update_problem(UpdateProblemArgs(problem_id, problem_content, model))
@ -247,8 +268,8 @@ def delete_embedding_by_paragraph_ids(paragraph_ids: List[str]):
def update_embedding_document_id(paragraph_id_list, target_document_id, target_knowledge_id,
target_embedding_model_id=None):
target_embedding_model = get_embedding_model(
target_embedding_model_id) if target_embedding_model_id is not None else None
target_embedding_model = scoped_model(
target_embedding_model_id, target_knowledge_id) if target_embedding_model_id is not None else None
ListenerManagement.update_embedding_document_id(
UpdateEmbeddingDocumentIdArgs(paragraph_id_list, target_document_id, target_knowledge_id,
target_embedding_model))

View File

@ -0,0 +1,146 @@
# coding=utf-8
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
from django.db import transaction
from django.utils import timezone
from common.db.sql_execute import sql_execute
from common.event.listener_manage import ListenerManagement
from common.utils.logger import maxkb_logger
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
TaskType, State, Status)
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
from ops import celery_app
POLL_SECONDS = 5
MAX_POLLS = 120
def _index_exists(knowledge_id: str) -> bool:
rows = sql_execute(
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
[f"embedding_hnsw_idx_{knowledge_id}"]
)
return bool(rows)
def _fail(task_id, message):
KnowledgeRebuildTask.objects.filter(
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
).update(status=RebuildStatus.FAILED, error_message=message,
finished_at=timezone.now(), update_time=timezone.now())
@celery_app.task(name="celery:rebuild_knowledge_index")
def rebuild_knowledge_index(task_id: str):
# 重复投递不能重复清空向量或重复派发。
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
attempt=0, update_time=timezone.now())
if not claimed:
return
try:
task = KnowledgeRebuildTask.objects.get(id=task_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if knowledge is None:
raise ValueError("知识库不存在")
snapshot = task.config_snapshot or {}
model_id = snapshot.get("embedding_model_id")
if not model_id or str(knowledge.embedding_model_id) != model_id:
raise ValueError("向量模型已变更,请重新发起任务")
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
raise ValueError('嵌入参数已变更,请重新发起任务')
# 兼容历史任务;新任务在受理时即固定文档范围。
document_ids = snapshot.get("document_ids")
if document_ids is None:
document_ids = [str(pk) for pk in Document.objects.filter(
knowledge_id=knowledge.id).values_list("id", flat=True)]
snapshot["document_ids"] = document_ids
task.config_snapshot = snapshot
task.save(update_fields=["config_snapshot", "update_time"])
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
if not documents or len(documents) != len(document_ids):
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
raise ValueError("文档正在向量化,请完成后重新发起")
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
drop_knowledge_index(knowledge_id=str(knowledge.id))
task.document_total = len(document_ids)
task.save(update_fields=["document_total", "update_time"])
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
ListenerManagement.update_status(
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
TaskType.EMBEDDING, State.PENDING)
from knowledge.task.embedding import embedding_by_document
dispatch_errors = []
for document in documents:
try:
embedding_by_document.delay(str(document.id), model_id)
except Exception as exc:
ListenerManagement.update_status(
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
dispatch_errors.append(f"{document.name}:{exc}")
if dispatch_errors:
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
task.save(update_fields=["error_message", "update_time"])
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
except Exception as exc:
_fail(task_id, f"重建任务启动失败:{exc}")
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
@celery_app.task(name="celery:finalize_rebuild_task")
def finalize_rebuild_task(task_id: str):
try:
with transaction.atomic():
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
if task is None or task.status != RebuildStatus.RUNNING:
return
snapshot = task.config_snapshot or {}
document_ids = snapshot.get("document_ids", [])
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
success = states.count(State.SUCCESS)
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
for state in states) + len(document_ids) - len(documents)
pending = len(document_ids) - success - failed
task.document_total = len(document_ids)
task.document_success = success
task.document_failed = failed
if pending:
task.attempt += 1
if task.attempt < MAX_POLLS:
task.save(update_fields=["document_total", "document_success", "document_failed",
"attempt", "update_time"])
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
(task_id,), countdown=POLL_SECONDS))
return
task.status = RebuildStatus.FAILED
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
else:
knowledge_id = str(task.knowledge_id)
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
raise ValueError("向量模型已变更,请重新发起任务")
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
create_knowledge_index(knowledge_id=knowledge_id)
index_ready = _index_exists(knowledge_id)
if success and failed == 0 and index_ready:
task.status = RebuildStatus.SUCCESS
task.error_message = ""
elif success and failed and index_ready:
task.status = RebuildStatus.PARTIAL
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
else:
task.status = RebuildStatus.FAILED
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
task.finished_at = timezone.now()
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
"error_message", "finished_at", "attempt", "update_time"])
except Exception as exc:
_fail(task_id, f"重建结果汇总失败:{exc}")
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)

View File

@ -0,0 +1,3 @@
"""Celery 任务发现入口,确保独立重建和向量化任务被 Worker 注册。"""
from knowledge.task import embedding # noqa: F401
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task # noqa: F401

View File

@ -0,0 +1,83 @@
import unittest
from types import SimpleNamespace
from knowledge.embedding_params import ConfiguredEmbedding, validate_params, knowledge_params
class RecordingModel:
batch_size = 10
def __init__(self):
self.calls = []
def embed_query(self, text):
self.calls.append(('query', text))
return [3., 4.]
def embed_documents(self, texts):
self.calls.append(('documents', texts))
return [[3., 4.] for _ in texts]
class EmbeddingParamsTests(unittest.TestCase):
def test_inference_uses_separate_instructions_and_same_normalization(self):
base = RecordingModel()
model = ConfiguredEmbedding(base, {'normalize': True, 'query_prefix': '检索:', 'document_prefix': '正文:'})
self.assertEqual(model.embed_query('内容'), [.6, .8])
self.assertEqual(model.embed_documents(['内容']), [[.6, .8]])
self.assertEqual(base.calls, [('query', '检索:内容'), ('documents', ['正文:内容'])])
def test_batch_size_is_applied_to_real_calls(self):
base = RecordingModel()
model = ConfiguredEmbedding(base, {'batch_size': 2})
self.assertEqual(len(model.embed_documents(['a', 'b', 'c', 'd', 'e'])), 5)
self.assertEqual([len(call[1]) for call in base.calls], [2, 2, 1])
def test_knowledge_overrides_do_not_leak_between_shared_models(self):
base = RecordingModel()
first = ConfiguredEmbedding(base, {'query_prefix': 'A'})
second = ConfiguredEmbedding(base, {'query_prefix': 'B'})
first.embed_query('x'); second.embed_query('x'); first.embed_query('x')
self.assertEqual(base.calls, [('query', 'Ax'), ('query', 'Bx'), ('query', 'Ax')])
self.assertEqual(base.batch_size, 10)
def test_invalid_inputs_are_rejected(self):
for params in ([], {'batch_size': True}, {'batch_size': 0}, {'batch_size': 257},
{'batch_size': 2.5}, {'normalize': 'true'}, {'query_prefix': 1},
{'document_prefix': 'x' * 2001}, {'': 'value'}, {'custom': {}},
{'custom': ['value']}, {'x': 1, 'X': 2}, {'custom': float('nan')},
{'a' * 65: 'value'}, {str(i): 'value' for i in range(51)}):
with self.subTest(params=params), self.assertRaises(ValueError):
validate_params(params)
def test_custom_names_are_saved_but_unsupported_options_do_not_reach_inference(self):
params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'}
self.assertEqual(validate_params(params), params)
active = knowledge_params(SimpleNamespace(meta={'embedding_params': params}))
self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:', 'normalize': True})
base = RecordingModel()
model = ConfiguredEmbedding(base, active)
model.embed_query('问题')
self.assertEqual(base.calls, [('query', '检索:问题')])
def test_missing_normalization_defaults_to_unit_vectors(self):
self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {'normalize': True})
self.assertEqual(knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': None}})),
{'normalize': True})
self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {})
base = RecordingModel()
self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [.6, .8])
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents(['x']), [[.6, .8]])
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), [])
def test_explicit_false_keeps_original_vectors(self):
params = knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': False}}))
self.assertEqual(params, {'normalize': False})
model = ConfiguredEmbedding(RecordingModel(), params)
self.assertEqual(model.embed_query('x'), [3., 4.])
self.assertEqual(model.embed_documents(['x']), [[3., 4.]])
def test_zero_vector_is_safe(self):
base = RecordingModel()
base.embed_query = lambda _: [0., 0.]
self.assertEqual(ConfiguredEmbedding(base, {'normalize': True}).embed_query('x'), [0., 0.])

View File

@ -0,0 +1,293 @@
"""Real isolated persistence and document parsing regression checks."""
import io
import importlib
import unittest
from types import SimpleNamespace
from unittest.mock import patch, Mock
from django.test import TestCase
from django.core.files.uploadedfile import SimpleUploadedFile
from common.exception.app_exception import AppApiException
from common.utils.split_model import smart_split_paragraph, SplitModel
from common.utils.table_split import split_table_rows
from knowledge.models import KnowledgeFolder, Knowledge
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
class MetadataTests(TestCase):
def setUp(self):
KnowledgeFolder.objects.create(id='default', name='root')
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
meta={'internal': {'preserve': True}})
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
def test_metadata_persists_and_preserves_other_settings(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
self.assertEqual(self.k.meta['internal'], {'preserve': True})
result = Metadata.one('default', str(self.k.id))
self.assertEqual(len(result['fields']), 1)
self.assertEqual(result['updated_by'], '测试员')
self.assertTrue(result['updated_at'])
def test_partial_metadata_update_does_not_clear_other_fields(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
old = Metadata.one('default', str(self.k.id))
self.assertEqual(old['fields'][0]['name'], '业务领域')
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
self.k.refresh_from_db()
self.assertEqual(self.k.meta['internal'], {'preserve': True})
def test_custom_field_validation_does_not_write(self):
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
[{'name': str(i), 'value': ''} for i in range(51)]]
for fields in cases:
with self.subTest(fields=fields), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
with self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
from users.models import User
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
original_time = self.k.create_time
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.k.refresh_from_db()
self.assertEqual(self.k.user_id, creator.id)
self.assertEqual(self.k.create_time, original_time)
self.assertEqual(result['creator_name'], '新创建者')
self.assertEqual(result['user_id'], str(creator.id))
self.assertTrue(result['update_time'])
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
from users.models import User, UserActivationApplication
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
for status in ('PENDING', 'REJECTED'):
application.status = status; application.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
application.status = 'APPROVED'; application.save()
creator.is_active = False; creator.save()
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
self.k.refresh_from_db()
self.assertIsNone(self.k.user_id)
self.assertNotIn('business_metadata', self.k.meta)
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
from knowledge.serializers.knowledge import KnowledgeSerializer
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
with patch.object(operation, 'is_valid'):
operation.edit({'embedding_params': params}, select_one=False)
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
self.k.refresh_from_db(); other.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
self.assertNotIn('embedding_params', other.meta)
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['embedding_params'], params)
def test_general_settings_cannot_restore_stale_metadata(self):
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
self.k.refresh_from_db()
protected = self.k.meta['business_metadata']
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
operation._validated_data = operation.initial_data
operation._errors = {}
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
self.k.refresh_from_db()
self.assertEqual(self.k.meta['business_metadata'], protected)
self.assertEqual(self.k.meta['selector'], 'body')
self.assertNotIn('parser_id', self.k.meta)
def test_invalid_types_and_unknown_fields_do_not_write(self):
for data in ([1], {'business_domain': 123}, {'business_domain': False},
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
with self.subTest(data=data), self.assertRaises(AppApiException):
Metadata.update('default', str(self.k.id), self.user, data)
self.k.refresh_from_db()
self.assertNotIn('business_metadata', self.k.meta)
def test_workspace_boundary(self):
for fn in (Metadata.one, Rules.one):
with self.assertRaises(AppApiException):
fn('other', str(self.k.id))
def test_segmentation_and_metadata_updates_preserve_each_other(self):
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
def test_segmentation_rejects_invalid_and_fractional_values(self):
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
(100,1.2), (True,0), (100,False), ('100.5',0)):
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
from knowledge.serializers.document import DocumentSerializers
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_size'], 500)
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
serializer.parse({'file':[SimpleUploadedFile('default.txt',b'test')]})
self.assertEqual(split.call_args.args[-2:], (500,0))
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
f=SimpleUploadedFile('test.txt',b'test')
serializer.parse({'file':[f]})
self.assertEqual(split.call_args.args[-2:], (100,20))
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
self.assertEqual(split.call_args.args[-2:], (200,0))
with self.assertRaises(AppApiException):
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
class ParserTests(unittest.TestCase):
text=''.join(chr(0x4e00+i) for i in range(500))
def check_overlap(self, parts):
self.assertGreater(len(parts),1)
self.assertTrue(all(len(p)<=100 for p in parts))
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
def test_plain_text_overlap(self):
self.check_overlap(smart_split_paragraph(self.text,100,20))
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
result=PdfSplitHandle.handle_toc(doc,100,20)
self.check_overlap([p['content'] for p in result])
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
import ast,inspect,textwrap
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
self.assertTrue(calls)
self.assertTrue(all(len(n.args)==3 for n in calls))
def test_table_overlap_preserves_rows_headers_and_termination(self):
header='| id | value |\n| --- | --- |\n'
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
chunks=split_table_rows(header,iter(rows),120,20)
self.assertGreater(len(chunks),1)
self.assertLess(len(chunks),len(rows))
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
def test_oversized_table_row_is_not_truncated(self):
row='| '+'x'*500+' |\n'
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
self.assertTrue(any(row in p['content'] for p in result))
self.assertIn('last',result[-1]['content'])
def test_csv_and_xlsx_use_row_overlap(self):
import openpyxl
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
book=openpyxl.Workbook();sheet=book.active
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
out=io.BytesIO();book.save(out)
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
f=SimpleUploadedFile(name,data)
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
chunks=result[0]['content'] if isinstance(result,list) else result['content']
self.assertGreater(len(chunks),1)
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_xls_uses_row_overlap(self):
from common.handle.impl.text.xls_split_handle import handle_sheet
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
chunks=handle_sheet('test',sheet,120,20)['content']
for a,b in zip(chunks,chunks[1:]):
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
manager=Mock();manager.using.return_value.filter.return_value=[model]
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
model.save.assert_called_once()
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
import json
import httpx
from openai import OpenAI
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
from langchain_core.messages import HumanMessage
requests=[]
def transport(request):
body=json.loads(request.content);requests.append(body)
if body.get('stream'):
events=[]
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
events.append('data: '+json.dumps(item)+'\n\n')
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
text=''.join(events)+'data: [DONE]\n\n')
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
model=BaiLianChatModel.new_instance('LLM','test',{
'api_base':'http://example.invalid/v1','api_key':'test-only'},
temperature=0.23,top_p=0.42,max_tokens=10)
model.client=client.chat.completions
model.root_client=client
model.streaming=False
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
model.streaming=True
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
self.assertEqual(len(requests),2)
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
for req in requests:
self.assertEqual(req['temperature'],0.23)
self.assertEqual(req['top_p'],0.42)
def test_new_knowledge_tasks_use_isolated_queue(self):
from ops import celery_app
import knowledge.tasks
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
self.assertIn(name,celery_app.tasks)
route=celery_app.amqp.router.route({},name)
self.assertEqual(route['queue'].name,'uniskb_knowledge')

View File

@ -0,0 +1,187 @@
"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls."""
import unittest
import uuid
from types import SimpleNamespace
from unittest.mock import patch, Mock
from contextlib import ExitStack
from django.db import transaction, connection
from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS
from common.exception.app_exception import AppApiException
class RebuildTests(unittest.TestCase):
def setUp(self):
with connection.cursor() as cursor:
cursor.execute('DELETE FROM knowledge_rebuild_task')
self.kid = str(uuid.uuid4())
self.docid = str(uuid.uuid4())
self.modelid = str(uuid.uuid4())
self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test')
self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2')
self.stack = ExitStack()
self.addCleanup(self.stack.close)
def mock(name):
return self.stack.enter_context(patch(name))
self.k = mock('knowledge.serializers.rebuild.Knowledge.objects')
self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge
self.k.filter.return_value.first.return_value = self.knowledge
self.docs = mock('knowledge.serializers.rebuild.Document.objects')
self.docs.filter.return_value.values_list.return_value = [self.docid]
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document])
self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects')
self.models = mock('knowledge.serializers.rebuild.Model.objects')
self.models.filter.return_value.first.return_value = SimpleNamespace(
id=self.modelid, name='embedding', model_name='test-model')
self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay')
self.delay.return_value.id = 'queued-test'
self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async')
self.listener = mock('knowledge.task.rebuild.ListenerManagement')
self.drop = mock('knowledge.task.rebuild.drop_knowledge_index')
self.create_index = mock('knowledge.task.rebuild.create_knowledge_index')
self.index_exists = mock('knowledge.task.rebuild._index_exists')
self.index_exists.return_value = True
self.child = mock('knowledge.task.embedding.embedding_by_document.delay')
def create(self):
return KnowledgeRebuildSerializer.create('default', self.kid, None)
def task(self, status=RebuildStatus.RUNNING):
return KnowledgeRebuildTask.objects.create(
knowledge_id=self.kid, workspace_id='default', status=status, document_total=1,
config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]})
def test_dispatch_only_after_commit(self):
with transaction.atomic():
result = self.create()
self.delay.assert_not_called()
self.assertEqual(result['status'], RebuildStatus.PENDING)
self.delay.assert_called_once()
self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test')
def test_rollback_does_not_dispatch(self):
with self.assertRaises(ValueError):
with transaction.atomic():
self.create()
raise ValueError('rollback')
self.delay.assert_not_called()
self.assertFalse(KnowledgeRebuildTask.objects.exists())
def test_broker_failure_persists_failed_record(self):
self.delay.side_effect = RuntimeError('broker offline')
result = self.create()
self.assertEqual(result['status'], RebuildStatus.FAILED)
self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message)
def test_active_task_rejected_under_knowledge_lock(self):
self.task()
with self.assertRaises(AppApiException):
self.create()
self.k.select_for_update.assert_called_once()
self.assertEqual(KnowledgeRebuildTask.objects.count(), 1)
def test_empty_knowledge_rejected(self):
self.docs.filter.return_value.values_list.return_value = []
with self.assertRaises(AppApiException):
self.create()
self.assertFalse(KnowledgeRebuildTask.objects.exists())
def test_success_cannot_be_retried(self):
task = self.task(RebuildStatus.SUCCESS)
with self.assertRaises(AppApiException):
KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None)
def test_failed_retry_creates_new_record(self):
old = self.task(RebuildStatus.FAILED)
result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None)
self.assertNotEqual(result['id'], str(old.id))
self.assertEqual(KnowledgeRebuildTask.objects.count(), 2)
def test_duplicate_worker_delivery_does_not_delete_vectors(self):
task = self.task()
rebuild_knowledge_index.run(str(task.id))
self.listener.delete_embedding_by_knowledge.assert_not_called()
def test_worker_dispatches_independent_monitor(self):
task = self.task(RebuildStatus.PENDING)
rebuild_knowledge_index.run(str(task.id))
self.child.assert_called_once_with(self.docid, self.modelid)
self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid])
self.poll.assert_called_once()
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.RUNNING)
def test_document_dispatch_failure_is_counted(self):
task = self.task(RebuildStatus.PENDING)
self.child.side_effect = RuntimeError('cannot queue')
rebuild_knowledge_index.run(str(task.id))
self.listener.update_status.assert_any_call(
self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE)
self.document.status = 'nn3'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.assertEqual(task.document_failed, 1)
self.assertIn('cannot queue', task.error_message)
def test_success_requires_index(self):
task = self.task()
self.index_exists.return_value = False
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_successful_finalization(self):
task = self.task()
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.SUCCESS)
self.assertEqual(task.document_success, 1)
self.assertIsNotNone(task.finished_at)
def test_pending_persists_progress_and_schedules_next_poll(self):
task = self.task()
other = SimpleNamespace(id=uuid.uuid4(), status='nn0')
task.config_snapshot['document_ids'].append(str(other.id))
task.save()
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other])
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.RUNNING)
self.assertEqual(task.document_success, 1)
self.poll.assert_called_once()
def test_timeout_records_failure(self):
task = self.task()
task.attempt = MAX_POLLS - 1
task.save()
self.document.status = 'nn0'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.assertIn('超时', task.error_message)
def test_ignored_document_is_failure(self):
task = self.task()
self.document.status = 'nnn'
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.document_failed, 1)
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_deleted_document_is_not_silent_success(self):
task = self.task()
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([])
finalize_rebuild_task.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.document_failed, 1)
self.assertEqual(task.status, RebuildStatus.FAILED)
def test_model_change_fails_before_destructive_work(self):
task = self.task(RebuildStatus.PENDING)
self.knowledge.embedding_model_id = str(uuid.uuid4())
rebuild_knowledge_index.run(str(task.id))
task.refresh_from_db()
self.assertEqual(task.status, RebuildStatus.FAILED)
self.listener.delete_embedding_by_knowledge.assert_not_called()

View File

@ -0,0 +1,350 @@
import uuid
import json
from datetime import timedelta
from types import SimpleNamespace
from unittest import TestCase
from unittest.mock import patch
from django.db import transaction
from django.test import RequestFactory
from django.utils import timezone
from rest_framework.exceptions import ValidationError, NotFound
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
from knowledge import versions as v
from knowledge.models import (KnowledgeFolder, Knowledge, Document, Paragraph, Problem, ProblemParagraphMapping,
Tag, DocumentTag, Embedding, File, KnowledgeVersion, KnowledgeVersionLease)
from users.models import User
from models_provider.models import Model
from application.models import Application, ApplicationFolder, ApplicationVersion, ApplicationKnowledgeMapping
class VersionTests(TestCase):
def setUp(self):
self.atomic = transaction.atomic(); self.atomic.__enter__()
from django.db import connection
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
self.user = User.objects.create(username='CODEX_VERSION_TEST', nick_name='版本测试', role='ADMIN')
self.folder = KnowledgeFolder.objects.create(id='default', name='默认')
ApplicationFolder.objects.create(id='default', name='默认')
self.model = Model.objects.create(name='测试向量', model_name='test-embedding', provider='test',
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
self.root = Knowledge.objects.create(name='CODEX知识库', desc='原始', folder=self.folder, user=self.user,
embedding_model=self.model, meta={'embedding_params': {'batch_size': 8}, 'segmentation': {'chunk_size': 500}})
self.doc = Document.objects.create(knowledge=self.root, name='样例', char_length=8, status='nn2')
self.para = Paragraph.objects.create(knowledge=self.root, document=self.doc, content='旧答案:100元', position=1)
self.problem = Problem.objects.create(knowledge=self.root, content='价格多少')
ProblemParagraphMapping.objects.create(knowledge=self.root, document=self.doc, paragraph=self.para, problem=self.problem)
tag = Tag.objects.create(knowledge=self.root, key='部门', value='测试')
DocumentTag.objects.create(document=self.doc, tag=tag)
Embedding.objects.create(id=str(uuid.uuid4()), knowledge=self.root, document=self.doc, paragraph=self.para,
source_id=str(self.para.id), source_type='1', embedding='[1,0,0]', search_vector='price')
def tearDown(self):
transaction.set_rollback(True); self.atomic.__exit__(None, None, None)
def create(self, name='初版'):
return v.create_version('default', self.root.id, self.user, {'name': name})
def edit_embedding(self, values):
from knowledge.serializers.knowledge import KnowledgeSerializer
return KnowledgeSerializer.Operate(data={'user_id': str(self.user.id), 'workspace_id': 'default',
'knowledge_id': str(self.root.id)}).edit(values, select_one=False)
def test_embedding_guard_blocks_version_but_allows_unchanged_and_batch(self):
from common.exception.app_exception import AppApiException
self.create()
self.edit_embedding({'embedding_model_id': str(self.model.id), 'embedding_params': {'batch_size': 12}})
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_params': {'batch_size': 12, 'normalize': False}})
def test_embedding_guard_detects_current_and_published_assistants(self):
from common.exception.app_exception import AppApiException
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
app = Application.objects.create(name='助手', user=self.user)
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
self.edit_embedding({'embedding_model_id': str(self.model.id)})
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=other)
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
ApplicationVersion.objects.create(application=app, knowledge_setting={
'_knowledge_ids': [str(self.root.id), str(other.id)]})
ApplicationKnowledgeMapping.objects.filter(application=app, knowledge=other).delete()
with self.assertRaises(AppApiException):
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
ApplicationVersion.objects.create(application=app, knowledge_setting={'_knowledge_ids': [str(self.root.id)]})
self.edit_embedding({'embedding_model_id': str(self.model.id)})
def test_embedding_guard_detects_workflow_and_allows_single_reference(self):
from knowledge.embedding_guard import embedding_change_reason, ASSISTANT_REASON
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
node = {'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_id_list': [str(self.root.id)]}}}
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user,
work_flow={'nodes': [node]})
self.assertEqual(embedding_change_reason(self.root), '')
node['properties']['node_data']['knowledge_id_list'].append(str(other.id))
app.work_flow = {'nodes': [node]}; app.save()
self.assertEqual(embedding_change_reason(self.root), ASSISTANT_REASON)
def test_embedding_guard_allows_model_change_with_single_assistant(self):
other_model = Model.objects.create(name='备用向量', model_name='test-embedding-2', provider='test',
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
app = Application.objects.create(name='助手', user=self.user)
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
self.edit_embedding({'embedding_model_id': str(other_model.id)})
self.root.refresh_from_db()
self.assertEqual(self.root.embedding_model_id, other_model.id)
def test_full_snapshot_and_id_remapping(self):
version = self.create()
self.assertTrue(version.search_ready)
self.assertEqual(version.document_count, 1)
paragraph = Paragraph.objects.get(knowledge=version.snapshot)
self.assertNotEqual(paragraph.id, self.para.id)
self.assertEqual(paragraph.content, self.para.content)
mapping = ProblemParagraphMapping.objects.get(knowledge=version.snapshot)
self.assertEqual(mapping.paragraph_id, paragraph.id)
self.assertEqual(mapping.document_id, paragraph.document_id)
self.assertEqual(DocumentTag.objects.filter(document__knowledge=version.snapshot).count(), 1)
vector = Embedding.objects.get(knowledge=version.snapshot)
self.assertEqual(vector.paragraph_id, paragraph.id)
self.assertEqual(vector.source_id, str(paragraph.id))
def test_live_edits_do_not_change_snapshot(self):
version = self.create()
self.para.content = '新答案:200元'; self.para.save()
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, '旧答案:100元')
second = self.create('第二版')
self.assertEqual(Paragraph.objects.get(knowledge=second.snapshot).content, '新答案:200元')
self.assertNotEqual(version.content_hash, second.content_hash)
def test_text_uuid_is_preserved(self):
self.para.content = '业务标识:' + str(self.doc.id); self.para.save()
version = self.create()
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, self.para.content)
def test_binary_files_shared_and_markdown_rewritten(self):
file = File(id=uuid.uuid4(), file_name='测试.png', loid=12345, source_id=str(self.root.id),
source_type='KNOWLEDGE', sha256_hash='test', file_size=10, meta={'knowledge_id': str(self.root.id)})
File.objects.bulk_create([file])
self.para.content = '![](./oss/file/' + str(file.id) + ')'; self.para.save()
version = self.create()
cloned = File.objects.get(source_id=str(version.snapshot_id))
self.assertEqual(cloned.loid, file.loid)
self.assertNotEqual(cloned.id, file.id)
self.assertIn(str(cloned.id), Paragraph.objects.get(knowledge=version.snapshot).content)
with patch('knowledge.models.knowledge.select_one') as unlink:
v.delete_version('default', self.root.id, version.id)
unlink.assert_not_called()
self.assertTrue(File.objects.filter(pk=file.id).exists())
def test_version_delete_cleans_only_snapshot(self):
version = self.create(); snapshot_id = version.snapshot_id
v.delete_version('default', self.root.id, version.id)
for model in (Document, Paragraph, Problem, Embedding, Tag, ProblemParagraphMapping):
self.assertFalse(model.objects.filter(knowledge_id=snapshot_id).exists())
self.assertFalse(Knowledge.objects.filter(pk=snapshot_id).exists())
self.assertTrue(Paragraph.objects.filter(pk=self.para.id).exists())
def test_version_number_not_reused(self):
first = self.create(); v.delete_version('default', self.root.id, first.id)
self.assertEqual(self.create().number, 2)
def test_create_transaction_rolls_back_on_index_failure(self):
with patch.object(v, '_build_index', side_effect=RuntimeError('index failed')), self.assertRaises(RuntimeError): self.create()
self.assertEqual(Knowledge.objects.count(), 1)
self.assertEqual(KnowledgeVersion.objects.count(), 0)
def test_pending_processing_rejected(self):
self.doc.status = 'nn0'; self.doc.save()
with self.assertRaises(ValidationError): self.create()
def test_missing_vectors_are_browsable_but_not_selectable(self):
Embedding.objects.all().delete(); version = self.create()
self.assertFalse(version.search_ready)
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [self.root.id])
def test_cross_knowledge_selection_rejected(self):
version = self.create()
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [])
def test_invalid_version_rejected(self):
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): 'invalid'}, 'default', [self.root.id])
def test_cross_workspace_rejected(self):
version = self.create()
with self.assertRaises(NotFound404): v.get_version('other', self.root.id, version.id)
def test_fixed_selection_and_live_lookup(self):
version = self.create()
self.assertEqual(v.resolve_search([self.root.id], {}, 'default'), [str(self.root.id)])
self.assertEqual(v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default'), [str(version.snapshot_id)])
def test_document_scope_maps_to_frozen_documents(self):
version = self.create(); document = Document.objects.get(knowledge=version.snapshot)
self.assertEqual(v.resolve_document_scope([self.doc.id], {str(self.root.id): str(version.id)}, 'default'), [str(document.id)])
self.assertEqual(v.resolve_document_scope([self.doc.id], {}, 'default'), [str(self.doc.id)])
data = v.version_data(version)
self.assertNotIn('_document_ids', data['config'])
self.assertNotIn('model_identity', data['config'])
def test_same_chat_keeps_latest_choice(self):
first = self.create()
selection = {str(self.root.id): 'latest'}
pinned = v.pin_selection(selection, 'default', 'chat-test')
self.create('新版')
self.assertEqual(v.pin_selection(selection, 'default', 'chat-test'), pinned)
self.assertEqual(pinned[str(self.root.id)], str(first.id))
self.assertNotEqual(v.pin_selection(selection, 'default', 'new-chat'), pinned)
def test_active_conversation_protects_delete(self):
version = self.create(); v.pin_selection({str(self.root.id): str(version.id)}, 'default', 'chat-test')
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
KnowledgeVersionLease.objects.update(expires_at=timezone.now()-timedelta(seconds=1))
v.delete_version('default', self.root.id, version.id)
def test_assistant_pin_protects_delete(self):
version = self.create()
app = Application.objects.create(name='测试助手', user=self.user,
knowledge_setting={'version_selection': {str(self.root.id): str(version.id)}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
app.knowledge_setting = {}; app.save(); v.delete_version('default', self.root.id, version.id)
def test_latest_assistant_protects_only_latest_ready_version(self):
first = self.create(); second = self.create('新版')
Application.objects.create(name='测试助手', user=self.user,
knowledge_setting={'version_selection': {str(self.root.id): 'latest'}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, second.id)
v.delete_version('default', self.root.id, first.id)
def test_workflow_draft_and_published_version_protect_delete(self):
version = self.create()
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_id_list': [str(self.root.id)],
'knowledge_setting': {'version_selection': {str(self.root.id): str(version.id)}}}}}]}
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, work_flow=flow)
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
app.work_flow = {}; app.save()
published = ApplicationVersion.objects.create(application=app, type='WORK_FLOW', work_flow=flow)
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
published.delete(); v.delete_version('default', self.root.id, version.id)
def test_workflow_conflicting_versions_rejected(self):
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
'knowledge_setting': {'version_selection': {str(self.root.id): 'latest'}}}}}]}
with self.assertRaises(ValidationError):
v.application_selection({'version_selection': {str(self.root.id): 'different'}}, flow)
def test_republication_releases_superseded_version(self):
version = self.create(); app = Application.objects.create(name='重新发布', user=self.user)
ApplicationVersion.objects.create(application=app, knowledge_setting={
'version_selection': {str(self.root.id): str(version.id)}})
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
ApplicationVersion.objects.create(application=app, knowledge_setting={})
v.delete_version('default', self.root.id, version.id)
def test_migration_target_cannot_be_snapshot(self):
version = self.create(); request = RequestFactory().post('/document/migrate')
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(self.root.id),
'target_knowledge_id': str(version.snapshot_id)})
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
def test_published_chat_keeps_published_association(self):
from application.serializers.common import ChatInfo
version = self.create()
app = Application.objects.create(name='发布测试', user=self.user)
ApplicationVersion.objects.create(application=app, knowledge_setting={
'_knowledge_ids': [str(self.root.id)], 'version_selection': {str(self.root.id): str(version.id)}})
self.assertFalse(ApplicationKnowledgeMapping.objects.filter(application=app).exists())
chat = ChatInfo(str(uuid.uuid4()), str(self.user.id), 'PLATFORM_USER', [], [], str(app.id))
chat.get_application()
self.assertEqual(chat.knowledge_id_list, [str(self.root.id)])
self.assertEqual(chat.knowledge_version_map, {str(self.root.id): str(version.id)})
def test_feedback_body_cannot_write_snapshot(self):
from application.serializers.application_chat_record import ApplicationChatRecordAddKnowledgeSerializer
version = self.create(); doc = Document.objects.get(knowledge=version.snapshot)
app = Application.objects.create(name='改进测试', user=self.user)
serializer = ApplicationChatRecordAddKnowledgeSerializer(data={'workspace_id': 'default',
'application_id': str(app.id), 'knowledge_id': str(version.snapshot_id),
'document_id': str(doc.id), 'chat_ids': []})
with self.assertRaises(NotFound404): serializer.post_improve({}, with_valid=False)
def test_model_changes_fail_explicitly(self):
version = self.create(); self.model.model_name = 'different'; self.model.save()
with self.assertRaises(ValidationError): v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default')
def test_internal_snapshot_api_guard(self):
version = self.create(); request = RequestFactory().get('/admin/api/knowledge')
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(version.snapshot_id)})
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
with self.assertRaises(NotFound404): v.root_knowledge('default', version.snapshot_id)
def test_api_read_permission_and_workspace(self):
from rest_framework.test import APIRequestFactory, force_authenticate
from knowledge.views.version import KnowledgeVersionView
self.create(); factory = APIRequestFactory()
for roles, workspace, expected in [([], 'default', 403), (['ADMIN'], 'default', 200), (['ADMIN'], 'other', 404)]:
req = factory.get('/versions')
force_authenticate(req, user=self.user, token=SimpleNamespace(role_list=roles, permission_list=[]))
response = KnowledgeVersionView.as_view()(req, workspace_id=workspace, knowledge_id=self.root.id)
self.assertEqual(response.status_code, expected)
def test_empty_name_rejected(self):
with self.assertRaises(ValidationError): self.create(' ')
def test_real_postgres_retrieval_reads_frozen_content(self):
from django.db import connection
if connection.vendor != 'postgresql':
self.skipTest('Real pgvector retrieval requires isolated PostgreSQL schema')
from application.chat_pipeline.step.search_dataset_step.impl import base_search_dataset_step as search
version = self.create()
self.para.content = '新答案:200元'; self.para.save()
manage = SimpleNamespace(context={'knowledge_version_map': {str(self.root.id): str(version.id)}, 'chat_user_type': 'PLATFORM_USER'})
model = SimpleNamespace(model_type='EMBEDDING', name='测试向量')
embedding = SimpleNamespace(embed_query=lambda text: [1.0, 0.0, 0.0])
with patch.object(search, 'get_model_by_id', return_value=model), patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding):
result = search.BaseSearchDatasetStep().execute('价格', [str(self.root.id)], [], [], 3, 0.1,
search_mode='embedding', workspace_id='default', manage=manage)
self.assertEqual(len(result), 1)
self.assertEqual(result[0].content, '旧答案:100元')
self.assertIn('v1', result[0].knowledge_name)
def test_real_postgres_workflow_retrieves_frozen_document(self):
from django.db import connection
if connection.vendor != 'postgresql': self.skipTest('Real workflow pgvector retrieval requires PostgreSQL')
from application.flow.step_node.search_knowledge_node.impl import base_search_knowledge_node as search
version = self.create(); self.para.content = '新答案:200元'; self.para.save()
body = {'knowledge_version_map': {str(self.root.id): str(version.id)},
'chat_user_type': 'PLATFORM_USER', 'workspace_id': 'default'}
node = search.BaseSearchKnowledgeNode.__new__(search.BaseSearchKnowledgeNode)
node.context = {}; node.workflow_manage = SimpleNamespace(get_body=lambda: body,
get_reference_field=lambda *_: [str(self.doc.id)])
embedding = SimpleNamespace(embed_query=lambda _: [1.0, 0.0, 0.0])
with patch.object(search, 'get_model_instance_by_model_workspace_id'), \
patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding), \
patch.object(search.connection, 'close'):
result = node.execute([str(self.root.id)], {'top_n':3, 'similarity':0.1, 'search_mode':'embedding'},
'价格', True, 'referencing', 'document', ['previous', 'documents'])
self.assertEqual(result.node_variable['paragraph_list'][0]['content'], '旧答案:100元')
def test_api_create_browse_and_delete(self):
from rest_framework.test import APIRequestFactory, force_authenticate
from knowledge.views.version import KnowledgeVersionView
factory = APIRequestFactory()
def call(method, data=None, version_id=None):
request = getattr(factory, method)('/versions', data=data or {}, format='json')
force_authenticate(request, user=self.user, token=SimpleNamespace(role_list=['ADMIN'], permission_list=[]))
kw = {'workspace_id': 'default', 'knowledge_id': self.root.id}
if version_id: kw['version_id'] = version_id
return KnowledgeVersionView.as_view()(request, **kw)
created = call('post', {'name': '接口创建'})
self.assertEqual(json.loads(created.content)['code'], 200)
version_id = json.loads(created.content)['data']['id']
detail = call('get', version_id=version_id)
self.assertEqual(json.loads(detail.content)['data']['documents'][0]['name'], self.doc.name)
removed = call('delete', version_id=version_id)
self.assertEqual(json.loads(removed.content)['code'], 200)

View File

@ -1,10 +1,13 @@
from django.urls import path
from . import views
from .views.version import KnowledgeVersionView
app_name = "knowledge"
# @formatter:off
urlpatterns = [
path('workspace/<str:workspace_id>/knowledge/<uuid:knowledge_id>/versions', KnowledgeVersionView.as_view()),
path('workspace/<str:workspace_id>/knowledge/<uuid:knowledge_id>/versions/<uuid:version_id>', KnowledgeVersionView.as_view()),
path('workspace/knowledge/document/template/export', views.Template.as_view()),
path('workspace/knowledge/document/table_template/export', views.TableTemplate.as_view()),
path('workspace/<str:workspace_id>/knowledge', views.KnowledgeView.as_view()),
@ -17,6 +20,12 @@ urlpatterns = [
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/sync', views.KnowledgeView.SyncWeb.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/generate_related', views.KnowledgeView.GenerateRelated.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/embedding', views.KnowledgeView.Embedding.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/metadata', views.KnowledgeView.Metadata.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/parser', views.KnowledgeView.Parser.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/segmentation', views.KnowledgeView.Segmentation.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild', views.KnowledgeView.Rebuild.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild/<str:task_id>', views.KnowledgeView.RebuildTask.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild/<str:task_id>/retry', views.KnowledgeView.RebuildRetry.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/hit_test', views.KnowledgeView.HitTest.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export', views.KnowledgeView.Export.as_view()),
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export_zip', views.KnowledgeView.ExportZip.as_view()),

View File

@ -82,7 +82,7 @@ class BaseVectorStore(ABC):
data = {'document_id': document_id, 'paragraph_id': paragraph_id, 'knowledge_id': knowledge_id,
'is_active': is_active, 'source_id': source_id, 'source_type': source_type, 'text': text}
chunk_list = chunk_data(data)
result = sub_array(chunk_list)
result = sub_array(chunk_list, self._embedding_batch_size(embedding))
for child_array in result:
self._batch_save(child_array, embedding, lambda: False)
@ -96,13 +96,22 @@ class BaseVectorStore(ABC):
"""
self.save_pre_handler()
chunk_list = chunk_data_list(data_list)
result = sub_array(chunk_list)
result = sub_array(chunk_list, self._embedding_batch_size(embedding))
for child_array in result:
if not is_the_task_interrupted():
self._batch_save(child_array, embedding, is_the_task_interrupted)
else:
break
@staticmethod
def _embedding_batch_size(embedding):
"""向量批次大小:优先取模型参数 batch_size,默认 10。"""
try:
batch_size = int(getattr(embedding, 'batch_size', 0) or 0)
except (TypeError, ValueError):
batch_size = 0
return batch_size if batch_size > 0 else 10
@abstractmethod
def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: str, paragraph_id: str, source_id: str,
is_active: bool,

View File

@ -0,0 +1,331 @@
"""Version snapshots reuse existing retrieval storage without exposing another knowledge base."""
import copy
import hashlib
import json
from datetime import timedelta
import uuid_utils.compat as uuid
from django.db import connection, transaction
from django.db.models import Q, OuterRef, Subquery
from django.core.exceptions import ValidationError as DjangoValidationError
from django.utils import timezone
from rest_framework.exceptions import ValidationError
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph,
Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status)
SNAPSHOT_KEY = '_version_snapshot'
def root_knowledge(workspace, knowledge_id, lock=False):
query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY)
if lock:
query = query.select_for_update()
obj = query.first()
if obj is None:
raise NotFound404(404, '知识库不存在')
return obj
def version_data(version):
return {'id': str(version.id), 'number': version.number, 'name': version.name,
'description': version.description, 'creator_name': version.creator_name,
'create_time': version.create_time, 'document_count': version.document_count,
'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count,
'search_ready': version.search_ready, 'content_hash': version.content_hash,
'config': {key: value for key, value in version.config.items()
if not key.startswith('_') and key != 'model_identity'}}
def remap(value, mapping):
if isinstance(value, dict):
return {k: remap(v, mapping) for k, v in value.items()}
if isinstance(value, list):
return [remap(v, mapping) for v in value]
if isinstance(value, str):
if value in mapping:
return mapping[value]
# File links are embedded in Markdown, HTML and document metadata.
import re
return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
lambda m: mapping.get(m[0], m[0]), value)
return value
def copy_rows(model, source_rows, ids, overrides=None):
objects = []
for row in source_rows:
values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields}
for field in model._meta.concrete_fields:
key = field.attname
value = values[key]
if key.endswith('_id') or key == 'id':
values[key] = ids.get(str(value), value)
elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str):
import re
values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})',
lambda match: match[1] + str(ids.get(match[2], match[2])), value)
elif isinstance(value, (dict, list, str)):
values[key] = remap(value, ids)
values.update(overrides or {})
objects.append(model(**values))
return model.objects.bulk_create(objects, batch_size=500)
def _build_index(snapshot_id):
if connection.vendor != 'postgresql':
return
with connection.cursor() as cursor:
cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)])
row = cursor.fetchone()
if row:
# Identifier and predicate are generated UUIDs; dimension comes from pgvector.
name = 'embedding_hnsw_idx_' + str(snapshot_id)
cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw '
'((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) '
"WHERE knowledge_id='" + str(snapshot_id) + "'::uuid")
def create_version(workspace, knowledge_id, user, data):
name = str(data.get('name') or '').strip()
description = str(data.get('description') or '').strip()
if not name or len(name) > 128 or len(description) > 512:
raise ValidationError('版本名称须为1~128字,说明最多512字')
nested = connection.in_atomic_block
with transaction.atomic():
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
if nested:
cursor.execute('SHOW transaction_isolation')
if cursor.fetchone()[0] not in ('repeatable read', 'serializable'):
raise ValidationError('版本创建需使用独立的一致性事务')
else:
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
root = root_knowledge(workspace, knowledge_id, lock=True)
documents = list(Document.objects.filter(knowledge=root).order_by('id'))
if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED)
for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)):
raise ValidationError('文档正在处理,请完成后创建版本')
paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id'))
problems = list(Problem.objects.filter(knowledge=root).order_by('id'))
mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root))
tags = list(Tag.objects.filter(knowledge=root))
doc_tags = list(DocumentTag.objects.filter(document__knowledge=root))
vectors = list(Embedding.objects.filter(knowledge=root))
files = list(File.objects.filter(Q(source_id=str(root.id)) |
Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id))))
number = root.version_sequence + 1
latest = KnowledgeVersion.objects.filter(knowledge=root).first()
number = max(number, latest.number + 1 if latest else 1)
version_id, snapshot_id = uuid.uuid7(), uuid.uuid7()
ids = {str(root.id): str(snapshot_id)}
for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files):
ids.update({str(row.pk): str(uuid.uuid7()) for row in rows})
config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None,
'embedding_params': (root.meta or {}).get('embedding_params', {}),
'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name,
'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})}
model = root.embedding_model
if model:
config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name,
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}
content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents],
'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs],
'problems': [[str(p.id), p.content] for p in problems],
'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config}
digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest()
config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents}
metadata = copy.deepcopy(root.meta or {})
metadata.pop('_version_sequence', None)
metadata.pop('parser_id', None)
metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)})
snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150],
desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id,
embedding_model_id=root.embedding_model_id, scope=root.scope, type=0,
file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata)
copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0})
copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id})
copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id})
copy_rows(DocumentTag, doc_tags, ids)
# Preserve binary large-object references; only the last File reference frees the bytes.
cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)})
for file in cloned_files:
file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)}
File.objects.bulk_update(cloned_files, ['meta'], batch_size=500)
copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id})
for old, new in zip(vectors, copied_vectors):
new.source_id = ids.get(str(old.source_id), old.source_id)
Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500)
active_docs = {d.id for d in documents if d.is_active}
active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs}
indexed = {v.paragraph_id for v in vectors if v.is_active}
search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed)
_build_index(snapshot_id)
version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot,
number=number, name=name, description=description, creator=user,
creator_name=user.nick_name or user.username, document_count=len(documents),
paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready,
content_hash=digest, config=config)
root.version_sequence = number
root.save(update_fields=['version_sequence', 'update_time'])
return version
def get_version(workspace, knowledge_id, version_id, lock=False):
root_knowledge(workspace, knowledge_id)
query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id)
if lock:
query = query.select_for_update()
obj = query.first()
if obj is None:
raise NotFound404(404, '知识库版本不存在')
return obj
def validate_selection(selection, workspace, knowledge_ids):
if not isinstance(selection, dict) or len(selection) > 100:
raise ValidationError('版本选择无效')
permitted = {str(k) for k in knowledge_ids}
for root_id, chosen in selection.items():
if str(root_id) not in permitted:
raise ValidationError('只能为已关联的知识库选择版本')
if not chosen:
continue
if chosen == 'latest':
version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace,
search_ready=True).first()
else:
try:
version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True).first()
except (ValueError, DjangoValidationError):
version = None
if version is None:
raise ValidationError('请选择已就绪且属于该知识库的版本')
def application_selection(knowledge_setting, work_flow=None):
selection = dict((knowledge_setting or {}).get('version_selection') or {})
knowledge_ids = []
for node in (work_flow or {}).get('nodes', []):
if node.get('type') != 'search-knowledge-node':
continue
data = node.get('properties', {}).get('node_data', {})
knowledge_ids.extend(data.get('knowledge_id_list') or [])
for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items():
if root_id in selection and selection[root_id] != chosen:
raise ValidationError('同一工作流中的知识库须使用相同版本')
selection[root_id] = chosen
return selection, knowledge_ids
@transaction.atomic
def pin_selection(selection, workspace, chat_id, pinned=None):
pinned = dict(pinned or {})
for root_id, chosen in sorted((selection or {}).items()):
if root_id in pinned or not chosen:
continue
root_knowledge(workspace, root_id, lock=True)
existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id,
expires_at__gt=timezone.now()).select_related('version').first()
if existing:
pinned[str(root_id)] = str(existing.version_id)
continue
query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True)
version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first()
if version is None:
raise ValidationError('选定知识库版本不可用,请检查助手设置')
pinned[str(root_id)] = str(version.id)
for root_id, version_id in sorted(pinned.items()):
root_knowledge(workspace, root_id, lock=True)
version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id,
knowledge__workspace_id=workspace, search_ready=True).first()
if version is None:
raise ValidationError('会话使用的知识库版本已删除,请新建会话')
KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id),
defaults={'expires_at': timezone.now() + timedelta(minutes=35)})
return pinned
def resolve_search(knowledge_ids, pinned, workspace):
result = []
for root_id in map(str, knowledge_ids):
if root_id not in (pinned or {}):
root_knowledge(workspace, root_id)
result.append(root_id)
continue
version = get_version(workspace, root_id, pinned[root_id])
if not version.search_ready:
raise ValidationError('知识库版本索引不可用')
identity = version.config.get('model_identity')
model = version.snapshot.embedding_model
if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name,
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}):
raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置')
result.append(str(version.snapshot_id))
return result
def resolve_document_scope(document_ids, pinned, workspace):
if document_ids is None:
return None
mapping = {}
for root_id, version_id in (pinned or {}).items():
mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {}))
return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids]
@transaction.atomic
def delete_version(workspace, knowledge_id, version_id):
root = root_knowledge(workspace, knowledge_id, lock=True)
version = get_version(workspace, knowledge_id, version_id, lock=True)
from application.models import Application, ApplicationVersion
key = 'knowledge_setting__version_selection__' + str(root.id)
is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first()
# Public chat uses only the latest publication. Superseded application history
# must not prevent users from retiring an otherwise unused knowledge version.
publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter(
application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1]))
for query in (Application.objects.all(), publications):
refs = Q(**{key: str(version.id)})
if is_latest and is_latest.id == version.id:
refs |= Q(**{key: 'latest'})
if query.filter(refs).exists():
raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布')
for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator():
for node in (flow or {}).get('nodes', []):
chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {})
.get('version_selection') or {}).get(str(root.id))
if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id):
raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布')
KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete()
if version.leases.exists():
raise ValidationError('版本有活跃会话,请会话结束后再删除')
snapshot_id = version.snapshot_id
Embedding.objects.filter(knowledge_id=snapshot_id).delete()
ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete()
DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete()
Tag.objects.filter(knowledge_id=snapshot_id).delete()
Paragraph.objects.filter(knowledge_id=snapshot_id).delete()
Document.objects.filter(knowledge_id=snapshot_id).delete()
Problem.objects.filter(knowledge_id=snapshot_id).delete()
File.objects.filter(source_id=str(snapshot_id)).delete()
if connection.vendor == 'postgresql':
with connection.cursor() as cursor:
cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"')
version.delete()
Knowledge.objects.filter(pk=snapshot_id).delete()
def guard_snapshot_request(request):
match = getattr(request, 'resolver_match', None)
kwargs = match.kwargs if match else {}
knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)]
if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists():
raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看')
file_id = kwargs.get('file_id')
if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter(
pk=file_id, meta__has_key='_version_id').exists():
raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除')

View File

@ -194,7 +194,7 @@ class DocumentView(APIView):
return result.success(DocumentSerializers.Split(data={
'workspace_id': workspace_id,
'knowledge_id': knowledge_id,
}).parse(split_data))
}, context={'user': request.user, 'request': request}).parse(split_data))
class SplitPattern(APIView):
authentication_classes = [TokenAuth]

View File

@ -1,3 +1,4 @@
from system_manage.ai_hub_knowledge import parser_binding
from django.utils.translation import gettext_lazy as _
from drf_spectacular.utils import extend_schema
from rest_framework.request import Request
@ -14,6 +15,8 @@ from knowledge.api.knowledge import KnowledgeBaseCreateAPI, KnowledgeWebCreateAP
from knowledge.models import KnowledgeScope
from knowledge.serializers.common import get_knowledge_operation_object
from knowledge.serializers.knowledge import KnowledgeSerializer
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer, KnowledgeSegmentationSerializer
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
from models_provider.serializers.model_serializer import ModelSerializer
@ -250,6 +253,154 @@ class KnowledgeView(APIView):
data={'knowledge_id': knowledge_id, 'workspace_id': workspace_id, 'user_id': request.user.id}
).embedding())
class Metadata(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="知识库业务元数据", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
def get(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(KnowledgeMetadataSerializer.one(workspace_id, knowledge_id))
@extend_schema(methods=['PUT'], summary="修改知识库业务元数据", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
RoleConstants.ADMIN,
)
@log(menu='Knowledge Base', operate='Modify knowledge metadata',
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
def put(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(KnowledgeMetadataSerializer.update(
workspace_id, knowledge_id, request.user, request.data))
class Parser(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="知识库解析配置", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
def get(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(parser_binding(workspace_id, knowledge_id, request.user))
@extend_schema(methods=['PUT'], summary="修改知识库解析配置", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
RoleConstants.ADMIN,
)
@log(menu='Knowledge Base', operate='Modify knowledge parser',
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
def put(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(parser_binding(workspace_id, knowledge_id, request.user, request.data))
class Segmentation(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="知识库分段规则", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
def get(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id))
@extend_schema(methods=['PUT'], summary="修改知识库分段规则", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
@log(menu='Knowledge Base', operate='Modify knowledge segmentation',
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
def put(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(KnowledgeSegmentationSerializer.update(
workspace_id, knowledge_id, request.user, request.data))
class Rebuild(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['POST'], summary="独立重建知识库索引", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
@log(menu='Knowledge Base', operate='Rebuild index',
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
def post(self, request: Request, workspace_id: str, knowledge_id: str):
return result.success(
KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, str(request.user.id))
)
@extend_schema(methods=['GET'], summary="重建任务列表", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
def get(self, request: Request, workspace_id: str, knowledge_id: str):
current_page = int(request.query_params.get('current_page', 1))
page_size = int(request.query_params.get('page_size', 10))
return result.success(
KnowledgeRebuildSerializer.page(workspace_id, knowledge_id, current_page, page_size)
)
class RebuildTask(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['GET'], summary="重建任务详情", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
def get(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str):
return result.success(
KnowledgeRebuildSerializer.one(workspace_id, knowledge_id, task_id)
)
class RebuildRetry(APIView):
authentication_classes = [TokenAuth]
@extend_schema(methods=['POST'], summary="重试重建任务", tags=["Knowledge Base"])
@has_permissions(
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([RoleConstants.USER.get_workspace_role()],
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
)
@log(menu='Knowledge Base', operate='Retry rebuild index',
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
def post(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str):
return result.success(
KnowledgeRebuildSerializer.retry(workspace_id, knowledge_id, task_id, str(request.user.id))
)
class Export(APIView):
authentication_classes = [TokenAuth]

View File

@ -0,0 +1,55 @@
from rest_framework.views import APIView
from rest_framework.exceptions import ValidationError
from common.auth import TokenAuth
from common.auth.authentication import has_permissions
from common.constants.permission_constants import PermissionConstants as P, RoleConstants as R, ViewPermission, CompareConstants
from common.log.log import log
from common.result import result
from knowledge import versions
from knowledge.models import KnowledgeVersion, Document, Paragraph
def rights(action):
permission = P.KNOWLEDGE_READ if action == 'read' else P.KNOWLEDGE_EDIT
return (R.ADMIN, permission.get_workspace_knowledge_permission(),
permission.get_workspace_permission_workspace_manage_role(), R.WORKSPACE_MANAGE.get_workspace_role(),
ViewPermission([R.USER.get_workspace_role()], [P.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND))
class KnowledgeVersionView(APIView):
authentication_classes = [TokenAuth]
@has_permissions(*rights('read'))
def get(self, request, workspace_id, knowledge_id, version_id=None):
versions.root_knowledge(workspace_id, knowledge_id)
if not version_id:
return result.success([versions.version_data(v) for v in KnowledgeVersion.objects.filter(knowledge_id=knowledge_id)])
version = versions.get_version(workspace_id, knowledge_id, version_id)
if request.query_params.get('content') == 'paragraphs':
try:
page = max(1, int(request.query_params.get('page', 1)))
size = max(1, min(100, int(request.query_params.get('page_size', 20))))
except (ValueError, TypeError):
raise ValidationError('分页参数无效')
query = Paragraph.objects.filter(knowledge_id=version.snapshot_id)
document_id = request.query_params.get('document_id')
if document_id:
query = query.filter(document_id=document_id)
search = request.query_params.get('search', '').strip()
if search:
query = query.filter(content__icontains=search)
return result.success({'total': query.count(), 'records': list(query.order_by('document_id', 'position', 'id')
.values('id', 'document_id', 'document__name', 'title', 'content', 'position', 'is_active')[(page-1)*size:page*size])})
return result.success({**versions.version_data(version), 'documents': list(Document.objects.filter(
knowledge_id=version.snapshot_id).order_by('name', 'id').values('id', 'name', 'char_length', 'is_active'))})
@has_permissions(*rights('edit'))
@log(menu='Knowledge Base', operate='Create knowledge version')
def post(self, request, workspace_id, knowledge_id):
return result.success(versions.version_data(versions.create_version(workspace_id, knowledge_id, request.user, request.data)))
@has_permissions(*rights('edit'))
@log(menu='Knowledge Base', operate='Delete knowledge version')
def delete(self, request, workspace_id, knowledge_id, version_id):
versions.delete_version(workspace_id, knowledge_id, version_id)
return result.success(True)

View File

@ -4,3 +4,9 @@ from django.apps import AppConfig
class LocalModelConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'local_model'
def ready(self):
import os
if os.environ.get('SERVER_NAME') == 'local_model' and os.environ.get('MAXKB_RUNTIME_AUTOSTART', '1') == '1':
from local_model.runtime import start_runtime
start_runtime()

View File

@ -0,0 +1,172 @@
"""Weight owner for local resources; shared by both assistants through model IDs."""
import gc
import hashlib
import json
import logging
import threading
import time
from urllib.parse import urlsplit, urlunsplit
import requests
from django.db import close_old_connections
from common.model_runtime import ModelRuntime, loading_policy
logger = logging.getLogger('maxkb')
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
def driver_for(provider, model_type):
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
def model_spec(model_id):
from local_model.models import Model
from local_model.serializers.rsa_util import rsa_long_decrypt
try:
model = Model.objects.get(id=model_id, status='SUCCESS')
except Model.DoesNotExist:
raise ValueError('模型不存在或尚不可用')
driver = driver_for(model.provider, model.model_type)
if not driver:
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
class WeightAdapter:
@staticmethod
def _request(spec, route, data=None):
url = urlsplit(spec['credential'].get('api_base', ''))
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
raise ValueError('模型服务地址无效')
path = url.path.rstrip('/')
if path.endswith('/v1'):
path = path[:-3]
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
key = spec['credential'].get('api_key')
headers = {'Authorization': 'Bearer ' + key} if key else {}
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
response.raise_for_status()
return response.json()
def probe(self, spec):
try:
tags = self._request(spec, '/api/ps').get('models', [])
except requests.RequestException as exc:
raise ValueError('模型推理服务不可用') from exc
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
if item:
vram = item.get('size_vram', 0)
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
def load(self, spec):
close_old_connections()
try:
if spec['driver'] == 'ollama':
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
'keep_alive': keep_alive})
info = self.probe(spec)
if not info:
raise ValueError('模型未驻留')
logger.info('Model weights ready: %s', spec['id'])
return True, info
from local_model.serializers.model_apply_serializers import get_local_model
instance = get_local_model(spec['model'], use_local=True)
if spec['type'] == 'EMBEDDING':
instance.embed_query('模型加载检查')
else:
from langchain_core.documents import Document
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
info = {'device': device}
try:
import torch
if device.startswith('cuda'):
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
except ImportError:
pass
logger.info('Model weights ready: %s', spec['id'])
return instance, info
finally:
close_old_connections()
def unload(self, spec, value):
if spec['driver'] == 'ollama':
# Refuse to unload a native model shared by another configured resource.
from local_model.models import Model
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
if spec['type'] == 'EMBEDDING':
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
else:
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
for _ in range(50):
if self.probe(spec) is None:
return
time.sleep(.2)
raise ValueError('模型卸载未完成,请稍后重试')
@staticmethod
def collect():
gc.collect()
try:
import torch
if torch.cuda.is_initialized():
torch.cuda.empty_cache()
except ImportError:
pass
@staticmethod
def error(exc):
name = type(exc).__name__
logger.warning('Model weight load failed: %s', name)
if 'OutOfMemory' in name:
return '模型加载失败:显存或内存不足'
return '模型加载失败,请检查模型文件、运行设备及服务状态'
manager = ModelRuntime(model_spec, WeightAdapter())
_startup_lock = threading.Lock()
_started = False
def start_runtime():
global _started
with _startup_lock:
if _started:
return
_started = True
def run():
from django.apps import apps
while not apps.ready:
time.sleep(.05)
from local_model.models import Model
try:
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
manager.preload(ids)
logger.info('Startup model preloads queued: %s', len(ids))
except Exception as exc:
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
finally:
close_old_connections()
while True:
time.sleep(10)
try:
manager.sweep()
except Exception as exc:
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
finally:
close_old_connections()
threading.Thread(target=run, name='model-startup', daemon=True).start()

View File

@ -7,72 +7,14 @@
@desc:
"""
import json
import threading
import time
from django.db import connection
from django.db.models import QuerySet
from django.utils.translation import gettext_lazy as _
from langchain_core.documents import Document
from rest_framework import serializers
from local_model.models import Model
from local_model.serializers.rsa_util import rsa_long_decrypt
from models_provider.impl.local_model_provider.local_model_provider import LocalModelProvider
from common.cache.mem_cache import MemCache
_lock = threading.Lock()
locks = {}
class ModelManage:
cache = MemCache('model', {})
up_clear_time = time.time()
@staticmethod
def _get_lock(_id):
lock = locks.get(_id)
if lock is None:
with _lock:
lock = locks.get(_id)
if lock is None:
lock = threading.Lock()
locks[_id] = lock
return lock
@staticmethod
def get_model(_id, get_model):
model_instance = ModelManage.cache.get(_id)
if model_instance is None:
lock = ModelManage._get_lock(_id)
with lock:
model_instance = ModelManage.cache.get(_id)
if model_instance is None:
model_instance = get_model(_id)
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
else:
if model_instance.is_cache_model():
ModelManage.cache.touch(_id, timeout=60 * 60 * 8)
else:
model_instance = get_model(_id)
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
ModelManage.clear_timeout_cache()
return model_instance
@staticmethod
def clear_timeout_cache():
if time.time() - ModelManage.up_clear_time > 60 * 60:
threading.Thread(target=lambda: ModelManage.cache.clear_timeout_data()).start()
ModelManage.up_clear_time = time.time()
@staticmethod
def delete_key(_id):
if ModelManage.cache.has_key(_id):
ModelManage.cache.delete(_id)
def get_local_model(model, **kwargs):
return LocalModelProvider().get_model(model.model_type, model.model_name,
json.loads(
@ -81,19 +23,10 @@ def get_local_model(model, **kwargs):
streaming=True, **kwargs)
def get_embedding_model(model_id):
model = QuerySet(Model).filter(id=model_id).first()
# 手动关闭数据库连接
connection.close()
embedding_model = ModelManage.get_model(model_id,
lambda _id: get_local_model(model, use_local=True))
return embedding_model
class EmbedDocuments(serializers.Serializer):
texts = serializers.ListField(required=True, child=serializers.CharField(required=True,
label=_('vector text')),
label=_('vector text list')),
label=_('vector text list'))
class EmbedQuery(serializers.Serializer):
@ -133,22 +66,26 @@ class ModelApplySerializers(serializers.Serializer):
self.is_valid(raise_exception=True)
EmbedDocuments(data=instance).is_valid(raise_exception=True)
model = get_embedding_model(self.data.get('model_id'))
return model.embed_documents(instance.getlist('texts'))
from local_model.runtime import manager
with manager.inference(self.data.get('model_id')) as model:
texts = instance.getlist('texts') if hasattr(instance, 'getlist') else instance.get('texts')
return model.embed_documents(texts)
def embed_query(self, instance, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
EmbedQuery(data=instance).is_valid(raise_exception=True)
model = get_embedding_model(self.data.get('model_id'))
from local_model.runtime import manager
with manager.inference(self.data.get('model_id')) as model:
return model.embed_query(instance.get('text'))
def compress_documents(self, instance, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
CompressDocuments(data=instance).is_valid(raise_exception=True)
model = get_embedding_model(self.data.get('model_id'))
from local_model.runtime import manager
with manager.inference(self.data.get('model_id')) as model:
return [{'page_content': d.page_content, 'metadata': d.metadata} for d in model.compress_documents(
[Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document in
instance.get('documents')], instance.get('query'))]
@ -156,5 +93,6 @@ class ModelApplySerializers(serializers.Serializer):
def unload(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
ModelManage.delete_key(self.data.get('model_id'))
from local_model.runtime import manager
manager.unload(self.data.get('model_id'))
return True

View File

@ -3,10 +3,12 @@ import os
from django.urls import path
from . import views
from .views.runtime import LocalModelRuntime
app_name = "local_model"
# @formatter:off
urlpatterns = [
path('model_runtime', LocalModelRuntime.as_view()),
path('model/validate', views.LocalModelApply.Validate.as_view()),
path('model/<str:model_id>/embed_documents', views.LocalModelApply.EmbedDocuments.as_view()),
path('model/<str:model_id>/embed_query', views.LocalModelApply.EmbedQuery.as_view()),

View File

@ -35,8 +35,9 @@ class LocalModelApply(APIView):
class Unload(APIView):
def post(self, request: Request, model_id):
return result.success(
ModelApplySerializers(data={'model_id': model_id}).compress_documents(request.data))
# Administrative weight operations use the signed runtime endpoint.
from rest_framework.exceptions import PermissionDenied
raise PermissionDenied('请通过模型管理卸载')
class Validate(APIView):
def post(self, request: Request):

View File

@ -0,0 +1,48 @@
import re
from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.serializers import UUIDField
from rest_framework.views import APIView
from common.model_runtime_client import verify_request
from common.auth.authenticate import AnonymousAuthentication
from common.result import result
from local_model.runtime import manager
class LocalModelRuntime(APIView):
authentication_classes = [AnonymousAuthentication]
permission_classes = []
def post(self, request):
try:
data = verify_request(request.body, request.headers.get('X-Model-Runtime-Signature'))
except (ValueError, TypeError):
raise PermissionDenied('模型服务认证失败')
model_id = str(UUIDField().run_validation(data.get('model_id')))
operation = data.get('operation')
try:
if operation == 'release':
lease = data.get('lease', '')
if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{32}', lease):
raise ValueError('请求标识无效')
manager.release(model_id, lease)
return result.success({'released': True})
if operation == 'acquire':
lease = manager.acquire(model_id, remote=True)
try:
return result.success({**manager.status(model_id), 'lease': lease})
except Exception:
manager.release(model_id, lease)
raise
if operation == 'load':
manager.submit(model_id)
elif operation == 'unload':
manager.unload(model_id)
elif operation == 'configure':
manager.configure(model_id)
elif operation != 'status':
raise ValueError('模型服务操作无效')
return result.success(manager.status(model_id))
except (ValueError, TimeoutError) as exc:
raise ValidationError(str(exc))

View File

@ -46,7 +46,10 @@ class Config(dict):
}
def get_debug(self) -> bool:
return self.get('DEBUG') if 'DEBUG' in self else True
value = self.get('DEBUG', True)
if isinstance(value, str):
return value.strip().lower() in ('1', 'true', 'yes', 'on')
return bool(value)
def get_time_zone(self) -> str:
return self.get('TIME_ZONE') if 'TIME_ZONE' in self else 'Asia/Shanghai'

View File

@ -62,6 +62,7 @@ MIDDLEWARE = [
REST_FRAMEWORK = {
'EXCEPTION_HANDLER': 'common.exception.handle_exception.handle_exception',
'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
'UNAUTHENTICATED_USER': None,
'DEFAULT_AUTHENTICATION_CLASSES': ['common.auth.authenticate.AnonymousAuthentication']
}
STATICFILES_DIRS = [(os.path.join(PROJECT_DIR, 'ui', 'dist'))]

View File

@ -97,8 +97,11 @@ class MaxKBBaseModel(ABC):
@staticmethod
def filter_optional_params(model_kwargs):
from system_manage.summary_plan import LOCAL_PARAMETERS
optional_params = {}
for key, value in model_kwargs.items():
if key in LOCAL_PARAMETERS and key != 'timeout':
continue
if key not in ['model_id', 'use_local', 'streaming', 'show_ref_label', 'stream']:
if key == 'extra_body' and isinstance(value, dict):
optional_params = {**optional_params, **value}

View File

@ -0,0 +1,34 @@
"""Deployment defaults; explicit model settings always take precedence."""
DEFAULT_CONTEXT_TOKENS = 65536
DEFAULT_SUMMARY_OUTPUT_TOKENS = 8192
DEFAULT_SAFETY_TOKENS = 4096
def model_context(params):
value = params.get('context_tokens')
if value is None:
value = params.get('num_ctx', DEFAULT_CONTEXT_TOKENS)
if isinstance(value, str) and value.isdigit():
value = int(value)
if type(value) is not int or not 1024 <= value <= 2000000:
raise ValueError('模型上下文须为 1024 至 2000000 的整数')
return value
def summary_defaults(params):
result = dict(params)
context = model_context(result)
result['context_tokens'] = context
result.setdefault('safety_tokens', min(DEFAULT_SAFETY_TOKENS, max(256, context // 16)))
safety = int(result['safety_tokens'])
output = result.get('output_tokens', result.get('max_tokens'))
if output is None:
output = min(DEFAULT_SUMMARY_OUTPUT_TOKENS, max(256, context // 8),
int(result.get('max_output_tokens') or DEFAULT_SUMMARY_OUTPUT_TOKENS))
result['output_tokens'] = int(output)
result.setdefault('max_output_tokens', max(int(output), min(DEFAULT_SUMMARY_OUTPUT_TOKENS,
context - safety - 1024)))
result.setdefault('max_input_tokens', context)
result.setdefault('input_tokens', min(int(result['max_input_tokens']), context - int(output) - safety))
return result

View File

@ -25,6 +25,20 @@ class BaiLianLLMModelParams(BaseForm):
precision=2
)
top_p = forms.SliderField(
TooltipLabel(
_('TopP'),
_('Nucleus sampling: the model considers the results of the top_p probability mass of tokens. '
'The smaller the value, the more deterministic the generation')
),
required=True,
default_value=0.8,
_min=0,
_max=1,
_step=0.01,
precision=2
)
max_tokens = forms.SliderField(
TooltipLabel(
_('Output the maximum Tokens'),

View File

@ -14,6 +14,9 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI):
@staticmethod
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
# Client controls belong to the SDK, not the provider's JSON body.
client_params = {key: optional_params.pop(key) for key in
('temperature', 'top_p', 'max_tokens', 'timeout', 'max_retries') if key in optional_params}
# if 'qwen-omni-turbo' in model_name or 'qwq' in model_name:
# optional_params['streaming'] = True
return BaiLianChatModel(
@ -21,5 +24,7 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI):
openai_api_base=model_credential.get('api_base'),
openai_api_key=model_credential.get('api_key'),
streaming=True,
stream_usage=True,
**client_params,
extra_body=optional_params
)

View File

@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _, gettext
from common import forms
from common.exception.app_exception import AppApiException
from common.forms import BaseForm
from common.forms import BaseForm, TooltipLabel
from models_provider.base_model_provider import BaseModelCredential, ValidCode
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
@ -51,3 +51,29 @@ class LocalEmbeddingCredential(BaseForm, BaseModelCredential):
return model
cache_folder = forms.TextInputField(_('Model catalog'), required=True)
device = forms.SingleSelect(
TooltipLabel(_('Device'), _('Runtime device for the embedding model')),
text_field='label',
value_field='value',
option_list=[
{'value': 'cpu', 'label': 'CPU'},
{'value': 'cuda', 'label': 'GPU(CUDA)'},
],
required=False,
default_value='cpu'
)
normalize = forms.SwitchField(
TooltipLabel(_('Normalize'), _('Whether to normalize embeddings to unit length')),
required=False,
default_value=True,
attrs={"active-value": "true", "inactive-value": "false"}
)
batch_size = forms.SliderField(
TooltipLabel(_('Batch size'), _('Number of texts per embedding request')),
required=False,
default_value=10,
_min=1,
_max=64,
_step=1,
precision=0
)

View File

@ -15,6 +15,7 @@ from models_provider.base_model_provider import MaxKBBaseModel
class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings):
batch_size: int = 10
@staticmethod
def is_cache_model():
return True
@ -27,7 +28,15 @@ class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings):
local_model_path = os.path.join(cache_folder, model_name)
if os.path.isdir(local_model_path):
model_path = local_model_path
return LocalEmbedding(model_name=model_path, cache_folder=cache_folder,
normalize = model_credential.get('normalize', True)
if isinstance(normalize, str):
normalize = normalize.lower() == 'true'
instance = LocalEmbedding(model_name=model_path, cache_folder=cache_folder,
model_kwargs={'device': model_credential.get('device')},
encode_kwargs={'normalize_embeddings': True}
encode_kwargs={'normalize_embeddings': normalize if normalize is not None else True}
)
try:
instance.batch_size = int(model_credential.get('batch_size') or 10)
except (TypeError, ValueError):
instance.batch_size = 10
return instance

View File

@ -24,19 +24,25 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings):
@staticmethod
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
try:
batch_size = int(model_credential.get('batch_size') or 10)
except (TypeError, ValueError):
batch_size = 10
return LocalEmbedding(model_name=model_name, cache_folder=model_credential.get('cache_folder'),
model_kwargs={'device': model_credential.get('device')},
encode_kwargs={'normalize_embeddings': True},
batch_size=batch_size,
**model_kwargs)
model_id: str = None
batch_size: int = 10
def embed_query(self, text: str) -> List[float]:
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
prefix = CONFIG.get_admin_path()
res = requests.post(
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_query',
{'text': text})
{'text': text}, timeout=(5, 620))
result = res.json()
if result.get('code', 500) == 200:
return result.get('data')
@ -47,7 +53,7 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings):
prefix = CONFIG.get_admin_path()
res = requests.post(
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_documents',
{'texts': texts})
{'texts': texts}, timeout=(5, 620))
result = res.json()
if result.get('code', 500) == 200:
return result.get('data')

View File

@ -8,6 +8,7 @@
"""
from typing import Sequence, Optional, Dict, Any
import os
from langchain_core.callbacks import Callbacks
from langchain_core.documents import Document, BaseDocumentCompressor
@ -39,7 +40,10 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor):
@staticmethod
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
return LocalReranker(model_name, cache_dir=model_credential.get('cache_dir'))
cache_dir = model_credential.get('cache_dir') or model_credential.get('cache_folder')
if cache_dir and not os.path.isabs(model_name) and os.path.isdir(os.path.join(cache_dir, model_name)):
model_name = os.path.join(cache_dir, model_name)
return LocalReranker(model_name, cache_dir=cache_dir, device=model_credential.get('device') or 'cpu')
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
Sequence[Document]:
@ -48,7 +52,7 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor):
import torch
with torch.no_grad():
inputs = self.tokenizer([[query, document.page_content] for document in documents], padding=True,
truncation=True, return_tensors='pt', max_length=512)
truncation=True, return_tensors='pt', max_length=512).to(self.client.device)
scores = [torch.sigmoid(s).float().item() for s in
self.client(**inputs, return_dict=True).logits.view(-1, ).float()]
result = [Document(page_content=documents[index].page_content, metadata={'relevance_score': scores[index]})

View File

@ -32,7 +32,6 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
print('ssss', kwargs.get('model_id', None))
self.model_id = kwargs.get('model_id', None)
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
@ -44,7 +43,7 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor):
res = requests.post(
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/compress_documents',
json={'documents': [{'page_content': document.page_content, 'metadata': document.metadata} for document in
documents], 'query': query}, headers={'Content-Type': 'application/json'})
documents], 'query': query}, headers={'Content-Type': 'application/json'}, timeout=(5, 620))
result = res.json()
if result.get('code', 500) == 200:
return [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document

View File

@ -12,7 +12,7 @@ from django.utils.translation import gettext as _
from common import forms
from common.exception.app_exception import AppApiException
from common.forms import BaseForm
from common.forms import BaseForm, TooltipLabel
from models_provider.base_model_provider import BaseModelCredential, ValidCode
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
@ -47,3 +47,12 @@ class OllamaEmbeddingModelCredential(BaseForm, BaseModelCredential):
return self
api_base = forms.TextInputField('API URL', required=True)
batch_size = forms.SliderField(
TooltipLabel(_('Batch size'), _('Number of texts per embedding request')),
required=False,
default_value=10,
_min=1,
_max=64,
_step=1,
precision=0
)

View File

@ -14,12 +14,26 @@ from models_provider.base_model_provider import MaxKBBaseModel
class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
model_resource_id: str | None = None
runtime_keep_alive: int = 300
batch_size: int = 10
@property
def _default_params(self):
return {**super()._default_params, 'keep_alive': self.runtime_keep_alive}
@staticmethod
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
return OllamaEmbedding(
try:
batch_size = int(model_credential.get('batch_size') or 10)
except (TypeError, ValueError):
batch_size = 10
instance = OllamaEmbedding(
model=model_name,
base_url=model_credential.get('api_base'),
model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None,
)
instance.batch_size = batch_size
return instance
def embed_documents(self, texts: List[str]) -> List[List[float]]:
"""Embed documents using an Ollama deployed embedding model.
@ -31,8 +45,13 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
List of embeddings, one for each text.
"""
instruction_pairs = [f"{text}" for text in texts]
embeddings = self._embed(instruction_pairs)
return embeddings
if not texts:
return []
from common.model_runtime_client import inference_lease
with inference_lease(self.model_resource_id) as state:
if state:
self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
return self._embed(instruction_pairs)
def embed_query(self, text: str) -> List[float]:
"""Embed a query using a Ollama deployed embedding model.
@ -44,5 +63,8 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
Embeddings for the text.
"""
instruction_pair = f"{text}"
embedding = self._embed([instruction_pair])[0]
return embedding
from common.model_runtime_client import inference_lease
with inference_lease(self.model_resource_id) as state:
if state:
self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
return self._embed([instruction_pair])[0]

View File

@ -26,6 +26,7 @@ def get_base_url(url: str):
class OllamaChatModel(MaxKBBaseModel, ChatOllama):
model_resource_id: str | None = None
@staticmethod
def is_cache_model():
return False
@ -35,10 +36,28 @@ class OllamaChatModel(MaxKBBaseModel, ChatOllama):
api_base = model_credential.get('api_base', '')
base_url = get_base_url(api_base)
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
from models_provider.context_policy import model_context
optional_params['num_ctx'] = model_context(model_kwargs)
return OllamaChatModel(model=model_name, base_url=base_url,
model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None,
stream=True, **optional_params)
def _create_chat_stream(self, messages, stop=None, **kwargs):
from common.model_runtime_client import inference_lease
with inference_lease(self.model_resource_id) as state:
if state:
kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
yield from super()._create_chat_stream(messages, stop, **kwargs)
async def _acreate_chat_stream(self, messages, stop=None, **kwargs):
from common.model_runtime_client import async_inference_lease
async with async_inference_lease(self.model_resource_id) as state:
if state:
kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
async for part in super()._acreate_chat_stream(messages, stop, **kwargs):
yield part
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
tokenizer = TokenizerManage.get_tokenizer()
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])

View File

@ -0,0 +1,258 @@
"""Model file assets, independent of runtime/model connection records. Never executes model code."""
import json
import hashlib
import os
import shutil
import stat
import tempfile
import uuid
import zipfile
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
MAX_BYTES = 2 * 1024 ** 3
MAX_FILES = 10000
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
class ImportError(ValueError):
pass
def model_fingerprint(directory):
"""Hash relative paths and complete file contents, independent of the archive name."""
root = Path(directory)
digest = hashlib.sha256()
for file in sorted((p for p in root.rglob('*') if p.is_file()), key=lambda p: p.relative_to(root).as_posix()):
name = file.relative_to(root).as_posix().encode('utf-8')
digest.update(len(name).to_bytes(8, 'big'))
digest.update(name)
digest.update(file.stat().st_size.to_bytes(8, 'big'))
with file.open('rb') as source:
for chunk in iter(lambda: source.read(1024 * 1024), b''):
digest.update(chunk)
return digest.hexdigest()
@contextmanager
def import_lock(root):
"""Serialize duplicate checks and record creation across Web processes."""
with (root / '.import.lock').open('a+b') as lock:
if os.name == 'nt':
import msvcrt
lock.seek(0)
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
else:
import fcntl
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
if os.name == 'nt':
lock.seek(0)
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
else:
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
def inspect_model(directory):
root = Path(directory)
if not root.is_dir() or root.is_symlink():
raise ImportError('请选择有效模型目录')
files = []
for current, dirs, names in os.walk(root, followlinks=False):
for name in dirs + names:
if (Path(current) / name).is_symlink():
raise ImportError('模型目录不能包含符号链接')
for name in names:
file = Path(current) / name
if not file.is_file():
raise ImportError('模型目录包含非普通文件')
files.append(file)
if len(files) > MAX_FILES:
raise ImportError('模型文件数量超过限制')
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
if not weights:
raise ImportError('未找到有效模型权重文件')
def read_json(name):
file = root / name
if not file.exists():
return {}
if file.stat().st_size > 2 * 1024 ** 2:
raise ImportError(name + '超过大小限制')
try:
data = json.loads(file.read_text(encoding='utf-8'))
if not isinstance(data, dict):
raise ValueError()
return data
except (ValueError, UnicodeError):
raise ImportError(name + '格式错误')
config = read_json('config.json')
manifest = read_json('model-manifest.json')
quant = config.get('quantization_config') or {}
if not isinstance(quant, dict):
raise ImportError('量化配置格式错误')
quantization = str(quant.get('quant_method', '')).lower()
suffixes = {f.suffix.lower() for f in weights}
if config and suffixes & {'.safetensors', '.bin'}:
if not config.get('model_type') and not config.get('architectures'):
raise ImportError('config.json缺少模型架构信息')
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
elif '.onnx' in suffixes:
format_name = 'ONNX'
elif '.gguf' in suffixes:
format_name = 'GGUF'
elif manifest:
format_name = 'CUSTOM'
else:
raise ImportError('模型目录缺少config.json或model-manifest.json')
# Validate referenced shards without trusting paths supplied by model files.
for index in root.glob('*.index.json'):
mapping = read_json(index.name).get('weight_map')
if not isinstance(mapping, dict) or not mapping:
raise ImportError('权重索引格式错误')
for name in mapping.values():
if not isinstance(name, str) or '\\' in name:
raise ImportError('权重分片路径无效')
relative = PurePosixPath(name)
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
raise ImportError('权重分片缺失或路径无效')
return {
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
'file_count': len(files), 'weight_count': len(weights),
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
'quantization': quantization,
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
'runtime_status': 'NOT_CHECKED',
'runtime_note': '文件已导入,运行能力需接入验证',
}
class ModelImportStore:
def __init__(self, root):
self.root = Path(root)
self.root.mkdir(parents=True, exist_ok=True)
def _records(self):
records = []
for path in self.root.glob('*/import.json'):
if path.parent.name.startswith('.import-'):
continue
try:
records.append(json.loads(path.read_text(encoding='utf-8')))
except (ValueError, OSError):
continue
return sorted(records, key=lambda r: r.get('created_at', ''), reverse=True)
def list(self):
records, paths, fingerprints = [], set(), set()
for record in self._records():
path = record.get('path')
canonical = os.path.normcase(os.path.realpath(path)) if path else None
fingerprint = record.get('fingerprint') if record.get('source') == 'UPLOAD' else None
if (canonical and canonical in paths) or (fingerprint and fingerprint in fingerprints):
continue
if canonical:
paths.add(canonical)
if fingerprint:
fingerprints.add(fingerprint)
records.append(record)
if len(records) == 200:
break
return records
def create(self, owner_id, path=None, upload=None):
if (path is None) == (upload is None):
raise ImportError('请选择一种导入方式')
identity = str(uuid.uuid4())
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
try:
if upload is not None:
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
raise ImportError('请上传2GB以内的ZIP模型包')
archive = staging / 'upload.zip'
with archive.open('wb') as output:
size = 0
for chunk in upload.chunks():
size += len(chunk)
if size > MAX_BYTES:
raise ImportError('模型包超过2GB')
output.write(chunk)
source = staging / 'files'
source.mkdir()
try:
with zipfile.ZipFile(archive) as zipped:
members = zipped.infolist()
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
seen = set()
for member in members:
name = PurePosixPath(member.filename)
mode = member.external_attr >> 16
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
or member.filename in seen):
raise ImportError('模型包包含不安全的路径或文件')
seen.add(member.filename)
target = source.joinpath(*name.parts)
if member.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
with zipped.open(member) as incoming, target.open('xb') as output:
shutil.copyfileobj(incoming, output, 1024 * 1024)
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
raise ImportError('模型包损坏或无法解压') from exc
archive.unlink()
# Most model archives contain one enclosing directory.
children = list(source.iterdir())
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
info = inspect_model(model_dir)
final_path = self.root / identity / model_dir.relative_to(staging)
source_type = 'UPLOAD'
display_name = Path(upload.name).stem
else:
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
raise ImportError('请输入服务器绝对目录')
model_dir = Path(path).resolve(strict=True)
info = inspect_model(model_dir)
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
with import_lock(self.root):
existing = self._records()
if source_type == 'PATH':
canonical = os.path.normcase(os.path.realpath(model_dir))
for record in existing:
if record.get('path') and os.path.normcase(os.path.realpath(record['path'])) == canonical:
return {**record, 'already_imported': True}
fingerprint = model_fingerprint(model_dir)
for record in existing:
if (record.get('size_bytes') != info['size_bytes'] or
record.get('file_count') != info['file_count']):
continue
previous_path = record.get('path')
if not previous_path or not Path(previous_path).is_dir():
continue
try:
inspect_model(previous_path)
previous = model_fingerprint(previous_path)
except (OSError, ImportError):
continue
if previous == fingerprint:
return {**record, 'already_imported': True}
record = {
'id': identity, 'name': display_name, 'path': str(final_path),
'source': source_type, 'owner_id': str(owner_id),
'created_at': datetime.now(timezone.utc).isoformat(),
'status': 'IMPORTED', 'fingerprint': fingerprint, **info,
}
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
staging.rename(self.root / identity)
return record
except (FileNotFoundError, PermissionError) as exc:
raise ImportError('目录不存在或无访问权限') from exc
finally:
if staging.exists():
shutil.rmtree(staging)

View File

@ -0,0 +1,28 @@
"""补齐已有百炼模型表单中的 TopP;保留全部自定义参数项。"""
from django.db import migrations
TOP_P_FIELD = {
"input_type": "Slider", "label": "TopP", "required": True,
"default_value": 0.8, "relation_show_field_dict": {},
"relation_trigger_field_dict": {}, "trigger_type": "OPTION_LIST",
"attrs": {"min": 0, "max": 1, "step": 0.01, "precision": 2,
"show-input-controls": False, "show-input": True},
"props_info": {}, "field": "top_p",
}
def add_top_p_params_form(apps, schema_editor):
Model = apps.get_model('models_provider', 'Model')
for model in Model.objects.using(schema_editor.connection.alias).filter(
provider='aliyun_bai_lian_model_provider', model_type='LLM'):
form = model.model_params_form or []
if any(item.get('field') == 'top_p' for item in form):
continue
model.model_params_form = [*form, TOP_P_FIELD]
model.save(update_fields=['model_params_form'], using=schema_editor.connection.alias)
class Migration(migrations.Migration):
dependencies = [('models_provider', '0001_initial')]
# 回退代码时保留参数项,不删除可能由管理员维护的数据。
operations = [migrations.RunPython(add_top_p_params_form, migrations.RunPython.noop)]

View File

@ -15,5 +15,5 @@ FROM (SELECT model."id"::text, model."name",
from model
left join "user" on user_id = "user".id
where model."id"::text in (select target
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
from effective_resource_permission ${workspace_user_resource_permission_query_set}
and 'VIEW' = any (permission_list)) ) temp ${model_query_set}

View File

@ -15,7 +15,7 @@ FROM (SELECT model."id"::text, model."name",
from model
left join "user" on user_id = "user".id
where model."id"::text in (select target
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
from effective_resource_permission ${workspace_user_resource_permission_query_set}
and case
when auth_type = 'ROLE' then
'ROLE' = any (permission_list)
@ -27,9 +27,9 @@ FROM (SELECT model."id"::text, model."name",
from role_permission role_permission
right join user_role_relation user_role_relation
on user_role_relation.role_id = role_permission.role_id
where user_role_relation.user_id = workspace_user_resource_permission.user_id
where user_role_relation.user_id = effective_resource_permission.user_id
and user_role_relation.workspace_id =
workspace_user_resource_permission.workspace_id)
effective_resource_permission.workspace_id)
else
'VIEW' = any (permission_list)

View File

@ -0,0 +1,74 @@
"""Context inheritance and native provider options; no network or database."""
from types import SimpleNamespace
from unittest import TestCase
from unittest.mock import patch
from models_provider.context_policy import model_context, summary_defaults
class ContextPolicyTests(TestCase):
def test_platform_budget_is_64k_with_room_for_output_and_prompt(self):
from system_manage.ai_hub_organize import resolve_settings
defaults = summary_defaults({})
self.assertEqual(defaults, {'context_tokens': 65536, 'max_input_tokens': 65536,
'input_tokens': 53248, 'max_output_tokens': 8192, 'output_tokens': 8192, 'safety_tokens': 4096})
self.assertEqual(resolve_settings({}, {}),
{'context_tokens': 65536, 'output_tokens': 8192, 'safety_tokens': 4096})
def test_explicit_model_limits_and_summary_budgets_win(self):
custom = {'context_tokens': '1000000', 'max_input_tokens': '1000000', 'input_tokens': '131072',
'max_output_tokens': '384000', 'output_tokens': '16384', 'safety_tokens': '4096'}
defaults = summary_defaults(custom)
self.assertEqual(defaults['context_tokens'], 1000000)
for key in custom.keys() - {'context_tokens', 'output_tokens'}:
self.assertEqual(defaults[key], custom[key])
self.assertEqual(defaults['output_tokens'], 16384)
self.assertEqual(model_context({'num_ctx': '131072'}), 131072)
def test_smaller_model_and_maximum_output_are_respected(self):
small = summary_defaults({'context_tokens': 4096, 'max_output_tokens': 256})
self.assertEqual(small['output_tokens'], 256)
self.assertLess(small['input_tokens'] + small['output_tokens'], 4096)
for value in (True, 0, -1, 1023, 2000001, 'invalid'):
with self.subTest(value=value), self.assertRaises(ValueError):
model_context({'context_tokens': value})
def test_only_llm_gets_default_context_from_model_configuration(self):
from models_provider.tools import get_model_default_params
model = SimpleNamespace(model_type='LLM', model_params_form=[])
self.assertEqual(get_model_default_params(model), {'context_tokens': 65536})
model.model_params_form = [{'field': 'num_ctx', 'default_value': '131072'}]
self.assertEqual(get_model_default_params(model)['context_tokens'], 131072)
model.model_type = 'EMBEDDING'
model.model_params_form = []
self.assertEqual(get_model_default_params(model), {})
def test_ollama_native_context_defaults_and_explicit_override(self):
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
for params, expected in (({}, 65536), ({'context_tokens': 131072}, 131072),
({'num_ctx': 32768}, 32768),
({'context_tokens': 131072, 'num_ctx': 32768}, 131072)):
with patch.object(OllamaChatModel, '__new__', return_value=SimpleNamespace()) as constructor:
OllamaChatModel.new_instance('LLM', 'local', {'api_base': 'http://localhost:11434'}, **params)
self.assertEqual(constructor.call_args.kwargs['num_ctx'], expected)
self.assertNotIn('context_tokens', constructor.call_args.kwargs)
def test_native_application_override_is_not_hidden_by_platform_default(self):
from models_provider import tools
model = SimpleNamespace(model_type='LLM', model_params_form=[])
with patch.object(tools, 'get_model_by_id', return_value=model), \
patch.object(tools.ModelManage, 'get_model', side_effect=lambda key, factory: factory(key)), \
patch.object(tools, 'get_model') as create:
tools.get_model_instance_by_model_workspace_id('test', 'default', num_ctx=131072)
self.assertEqual(create.call_args.kwargs['context_tokens'], 131072)
def test_metadata_and_preview_share_effective_budget_without_model_calls(self):
from system_manage.mcp_business import summary_model_metadata
from models_provider.tools import get_model_default_params
model = SimpleNamespace(id='test', name='test', model_type='LLM', provider='model_ollama_provider',
model_params_form=[])
with patch('models_provider.tools.get_model') as invoke:
metadata = summary_model_metadata(model)
invoke.assert_not_called()
self.assertEqual(metadata['summary_defaults']['input_tokens'], 53248)
self.assertEqual(metadata['summary_limits']['context_tokens'], 65536)

View File

@ -0,0 +1,158 @@
"""File import regression tests, temporary files only; no database or model runtimes."""
import io
import json
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest.mock import patch
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
class Upload:
def __init__(self, entries):
data = io.BytesIO()
with zipfile.ZipFile(data, 'w') as archive:
for name, content in entries:
archive.writestr(name, content)
self.body = data.getvalue()
self.size, self.name = len(self.body), 'model.zip'
def chunks(self):
yield self.body
class ImportTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.model = self.root / 'model'
self.model.mkdir()
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
self.store = ModelImportStore(self.root / 'imports')
def test_local_path_record_persists_without_copying_or_running(self):
record = self.store.create('owner', path=str(self.model))
self.assertEqual(record['format'], 'HF')
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
self.assertEqual(record['path'], str(self.model.resolve()))
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
def test_quantized_formats(self):
for method in ('gptq', 'awq'):
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
self.assertEqual(inspect_model(self.model)['format'], method.upper())
def test_onnx_gguf_custom(self):
(self.model / 'config.json').unlink()
(self.model / 'model.safetensors').unlink()
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
file = self.model / ('model' + suffix)
file.write_bytes(b'fixture')
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
self.assertEqual(inspect_model(self.model)['format'], expected)
file.unlink()
def test_zip_nested_model_and_persisted_path(self):
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
self.assertEqual(record['source'], 'UPLOAD')
def test_traversal_windows_path_rejected(self):
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
self.assertEqual(self.store.list(), [])
def test_duplicate_archive_entry_rejected(self):
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
def test_zip_symlink_rejected(self):
data=io.BytesIO()
with zipfile.ZipFile(data,'w') as archive:
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
archive.writestr(info, '../outside')
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
def test_size_limit_and_cleanup(self):
upload=Upload([('model.onnx', b'x'*100)])
with patch('models_provider.local_imports.MAX_BYTES', 20):
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
self.assertEqual(list(self.store.root.iterdir()), [])
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
(self.model / 'config.json').write_text('{broken')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'config.json').write_text('{"other":1}')
with self.assertRaises(ImportError):inspect_model(self.model)
(self.model / 'model.safetensors').unlink()
with self.assertRaises(ImportError):inspect_model(self.model)
def test_missing_or_escaping_shard_rejected(self):
for name in ('absent.safetensors','../outside.bin'):
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
with self.assertRaises(ImportError):inspect_model(self.model)
def test_bad_zip_and_missing_path_leave_no_records(self):
upload=Upload([]);upload.body=b'not zip';upload.size=7
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
self.assertEqual(self.store.list(), [])
def test_exclusive_source(self):
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
def test_repeated_path_and_historical_records_are_deduplicated(self):
first = self.store.create('owner', path=str(self.model))
repeated = self.store.create('owner', path=str(self.model / '..' / 'model'))
self.assertEqual(repeated['id'], first['id'])
self.assertTrue(repeated['already_imported'])
legacy = self.store.root / 'legacy'
legacy.mkdir()
(legacy / 'import.json').write_text(json.dumps({**first, 'id': 'legacy', 'created_at': '2000'}))
self.assertEqual([row['id'] for row in self.store.list()], [first['id']])
def test_zip_content_deduplicates_and_distinct_content_is_retained(self):
files = [('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]
first = self.store.create('owner', upload=Upload(files))
second = self.store.create('owner', upload=Upload(list(reversed(files))))
self.assertEqual(first['id'], second['id'])
self.assertTrue(second['already_imported'])
different = self.store.create('owner', upload=Upload([
('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights2')]))
self.assertNotEqual(first['id'], different['id'])
self.assertEqual(len(self.store.list()), 2)
def test_different_directory_and_zip_with_same_files_reuse_record(self):
first = self.store.create('owner', path=str(self.model))
copy = self.root / 'copy'
import shutil
shutil.copytree(self.model, copy)
second = self.store.create('owner', path=str(copy))
self.assertEqual(first['id'], second['id'])
zipped = Upload([('repo/config.json', (copy / 'config.json').read_bytes()),
('repo/model.safetensors', (copy / 'model.safetensors').read_bytes())])
self.assertEqual(self.store.create('owner', upload=zipped)['id'], first['id'])
def test_concurrent_imports_create_one_record(self):
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(lambda _: self.store.create('owner', path=str(self.model)), range(4)))
self.assertEqual(len({row['id'] for row in results}), 1)
self.assertEqual(len(self.store._records()), 1)
def test_invalid_legacy_directory_does_not_block_import(self):
first = self.store.create('owner', path=str(self.model))
config = self.model / 'config.json'
config.write_text('x' * len(config.read_text()))
replacement = self.root / 'replacement'
replacement.mkdir()
(replacement / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
(replacement / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
self.assertNotEqual(self.store.create('owner', path=str(replacement))['id'], first['id'])
if __name__ == '__main__':unittest.main()

View File

@ -0,0 +1,67 @@
import json
from types import SimpleNamespace
from unittest.mock import patch
from django.test import TestCase
from rest_framework.exceptions import ValidationError
from common.constants.permission_constants import RoleConstants
from common.exception.app_exception import AppUnauthorizedFailed
from models_provider.models import Model
from models_provider.views.runtime import ModelLoading
class LoadingApiTests(TestCase):
def setUp(self):
self.model = Model.objects.create(name='loading-test', provider='model_local_provider',
model_type='EMBEDDING', model_name='test-only', credential='test-only', meta={'existing': 'keep'})
self.admin = SimpleNamespace(auth=SimpleNamespace(role_list=[str(RoleConstants.ADMIN.value)], permission_list=[]), data={})
self.view = ModelLoading()
def args(self):
return {'workspace_id': 'default', 'model_id': str(self.model.id)}
def test_policy_is_persistent_and_preserves_existing_metadata(self):
self.admin.data = {'strategy': 'PRELOAD', 'priority': 90}
with patch('models_provider.views.runtime.runtime_request', return_value={'status': 'QUEUED'}) as rpc:
response = self.view.put(self.admin, **self.args())
self.model.refresh_from_db()
self.assertEqual(self.model.meta['existing'], 'keep')
self.assertEqual(self.model.meta['loading_policy'], {'strategy': 'PRELOAD', 'priority': 90, 'idle_seconds': 300})
self.assertEqual(json.loads(response.content)['data']['status'], 'QUEUED')
self.assertEqual(rpc.call_args_list[0].args[0], 'configure')
def test_normal_user_cannot_change_policy_or_load_weights(self):
user = SimpleNamespace(auth=SimpleNamespace(role_list=[], permission_list=[]), data={'action': 'load'})
for method in (self.view.get, self.view.put, self.view.post):
with self.assertRaises(AppUnauthorizedFailed):
method(user, **self.args())
def test_workspace_and_unsupported_provider_are_rejected(self):
with self.assertRaises(ValidationError):
self.view.get(self.admin, workspace_id='other', model_id=str(self.model.id))
self.model.provider = 'aliyun_bai_lian_model_provider'
self.model.model_type = 'LLM'; self.model.save()
response = self.view.get(self.admin, **self.args())
self.assertEqual(json.loads(response.content)['data']['status'], 'EXTERNAL')
with self.assertRaises(ValidationError):
self.view.put(self.admin, **self.args())
def test_arbitrary_actions_and_invalid_policies_do_not_reach_runtime(self):
with patch('models_provider.views.runtime.runtime_request') as rpc:
self.admin.data = {'action': 'shell'}
with self.assertRaises(ValidationError): self.view.post(self.admin, **self.args())
self.admin.data = {'priority': 500}
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
rpc.assert_not_called()
self.model.refresh_from_db()
self.assertNotIn('loading_policy', self.model.meta)
def test_saved_policy_remains_available_when_runtime_is_temporarily_down(self):
self.admin.data = {'strategy': 'PRELOAD', 'priority': 80}
with patch('models_provider.views.runtime.runtime_request', side_effect=ValueError('offline')):
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
response = self.view.get(self.admin, **self.args())
self.model.refresh_from_db()
self.assertEqual(self.model.meta['loading_policy']['priority'], 80)
self.assertEqual(json.loads(response.content)['data']['status'], 'UNAVAILABLE')

View File

@ -0,0 +1,199 @@
import contextlib
import json
import threading
import weakref
import unittest
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from unittest.mock import patch
from common.model_runtime import ModelRuntime, loading_policy
from common.model_runtime_client import signed_request, verify_request
class FakeAdapter:
def __init__(self):
self.loaded, self.unloaded = [], []
self.native_ready = True
self.fail = False
self.block = None
def load(self, spec):
if self.block:
self.block.wait(3)
if self.fail:
raise ValueError('private error details')
self.loaded.append(spec['id'])
self.native_ready = True
return SimpleNamespace(), {'device': 'cpu'}
def unload(self, spec, value):
self.unloaded.append(spec['id'])
self.native_ready = False
def probe(self, spec):
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
def collect(self):
pass
def error(self, exc):
return 'load failed'
class RuntimeTests(unittest.TestCase):
def setUp(self):
self.now = 0
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
for key, priority in [('low', 10), ('high', 90)]}
self.adapter = FakeAdapter()
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
self.adapter, lambda: self.now)
def test_no_load_until_first_inference_and_then_reuses_weights(self):
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
self.assertEqual(self.adapter.loaded, [])
with self.manager.inference('low') as first:
self.assertIsNotNone(first)
with self.manager.inference('low') as second:
self.assertIs(first, second)
self.assertEqual(self.adapter.loaded, ['low'])
def test_startup_preloads_highest_priority_first(self):
self.specs['low']['policy']['strategy'] = 'PRELOAD'
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['low', 'high'])
for key in ['low', 'high']:
self.manager.entries[key]['future'].result(3)
self.assertEqual(self.adapter.loaded, ['high', 'low'])
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
def test_concurrent_first_calls_only_load_once(self):
self.adapter.block = threading.Event()
with ThreadPoolExecutor(max_workers=6) as pool:
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
self.adapter.block.set()
tokens = [future.result(3) for future in futures]
self.assertEqual(len(set(tokens)), 6)
self.assertEqual(self.adapter.loaded, ['low'])
self.assertEqual(self.manager.status('low')['active_requests'], 6)
for token in tokens:
self.manager.release('low', token)
def test_in_use_models_cannot_be_unloaded(self):
token = self.manager.acquire('low')
with self.assertRaises(ValueError):
self.manager.unload('low')
self.manager.release('low', token)
self.manager.unload('low')
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
def test_unload_releases_the_last_loader_reference(self):
class Weights:
pass
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
token = self.manager.acquire('low')
pointer = weakref.ref(self.manager.entries['low']['value'])
self.manager.release('low', token)
self.manager.unload('low')
self.assertIsNone(pointer())
def test_idle_cleanup_respects_active_and_preloaded_models(self):
active = self.manager.acquire('low')
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['high'])
self.manager.entries['high']['future'].result(3)
self.now = 100
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, [])
self.manager.release('low', active)
self.now = 131
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, ['low'])
self.assertEqual(self.manager.status('high')['status'], 'READY')
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
self.adapter.fail = True
with self.assertRaisesRegex(ValueError, '^load failed$'):
self.manager.acquire('low')
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
self.adapter.fail = False
with self.manager.inference('low'):
pass
self.assertEqual(self.manager.status('low')['status'], 'READY')
def test_changed_model_config_never_reuses_old_weights(self):
token = self.manager.acquire('low')
self.specs['low']['fingerprint'] = 'v2'
with self.assertRaises(ValueError):
self.manager.acquire('low')
self.manager.release('low', token)
with self.manager.inference('low'):
pass
self.assertEqual(self.adapter.loaded, ['low', 'low'])
self.assertEqual(self.adapter.unloaded, ['low'])
def test_native_expiration_requires_loading_again(self):
self.specs['low']['driver'] = 'ollama'
token = self.manager.acquire('low')
self.manager.release('low', token)
self.adapter.native_ready = False
token = self.manager.acquire('low')
self.manager.release('low', token)
self.assertEqual(self.adapter.loaded, ['low', 'low'])
def test_policy_validation(self):
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
with self.subTest(value=value), self.assertRaises(ValueError):
loading_policy(value)
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
self.assertEqual(verify_request(body, signature)['operation'], 'load')
with self.assertRaises(ValueError):
verify_request(body.replace(b'load', b'unload'), signature)
with self.assertRaises(ValueError):
verify_request(body, '')
with patch('common.model_runtime_client.time.time', return_value=1):
body, signature = signed_request({'operation': 'load'})
with self.assertRaises(ValueError):
verify_request(body, signature)
class ProviderHookTests(unittest.TestCase):
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
events = []
@contextlib.contextmanager
def lease(model_id):
events.append(('acquire', model_id))
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append(('release', model_id))
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
self.assertEqual(model.embed_query('text'), [1., 2.])
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
self.assertEqual(model._default_params['keep_alive'], -1)
def test_llm_stream_releases_lease_when_generator_is_closed(self):
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
from langchain_ollama import ChatOllama
events = []
@contextlib.contextmanager
def lease(model_id):
events.append('acquire')
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append('release')
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
stream = model._create_chat_stream([])
next(stream); stream.close()
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
self.assertEqual(events, ['acquire', 'release'])

View File

@ -131,11 +131,15 @@ def get_model_default_params(model):
return value
return value
return {
params = {
p.get('field'): convert_to_int(p.get('default_value'))
for p in model.model_params_form
if p.get('default_value') is not None
}
if model.model_type == 'LLM':
from models_provider.context_policy import model_context
params['context_tokens'] = model_context(params)
return params
def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs):
@ -147,4 +151,8 @@ def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs):
"""
model = get_model_by_id(model_id, workspace_id)
s = get_model_default_params(model)
return ModelManage.get_model(model_id, lambda _id: get_model(model, **{**s, **kwargs}))
params = {**s, **kwargs}
if model.model_type == 'LLM' and any(kwargs.get(key) is not None for key in ('context_tokens', 'num_ctx')):
from models_provider.context_policy import model_context
params['context_tokens'] = model_context(kwargs)
return ModelManage.get_model(model_id, lambda _id: get_model(model, **params))

Some files were not shown because too many files have changed in this diff Show More