Compare commits
14 Commits
v3.2
...
v3.2-refac
| Author | SHA1 | Date |
|---|---|---|
|
|
d8e640748e | |
|
|
b23569e0b8 | |
|
|
79fa2cf4a2 | |
|
|
2600fbf3a4 | |
|
|
851cb6b484 | |
|
|
4aea21b6a1 | |
|
|
5ee44927b5 | |
|
|
91f9199020 | |
|
|
96a81496be | |
|
|
5434504f9b | |
|
|
2baba0b19c | |
|
|
e06b78918b | |
|
|
69eb1eb681 | |
|
|
6df950ec3f |
|
|
@ -190,3 +190,10 @@ apps/models_provider/impl/tencent_model_provider/model/stt.py
|
||||||
tmp/
|
tmp/
|
||||||
config.yml
|
config.yml
|
||||||
.SANDBOX_BANNED_HOSTS
|
.SANDBOX_BANNED_HOSTS
|
||||||
|
|
||||||
|
# Local verification outputs; release sources remain visible to Git.
|
||||||
|
/ui/dist*/
|
||||||
|
/ui/.summary-build/
|
||||||
|
/ui/.codex-validation*/
|
||||||
|
/jieba.cache
|
||||||
|
/mcp-engine-*/
|
||||||
|
|
|
||||||
|
|
@ -183,6 +183,10 @@ class BaseChatStep(IChatStep):
|
||||||
tool_ids=None,
|
tool_ids=None,
|
||||||
mcp_output_enable=True,
|
mcp_output_enable=True,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
|
from system_manage.mcp_agent import context_for
|
||||||
|
enabled = (model_setting or {}).get('engine_tools', [])
|
||||||
|
self.context['engine_body'] = {**manage.context, 'workspace_id': workspace_id, 'chat_id': str(chat_id)}
|
||||||
|
self.context['engine_tools'] = enabled
|
||||||
chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id,
|
chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id,
|
||||||
**model_params_setting) if model_id is not None else None
|
**model_params_setting) if model_id is not None else None
|
||||||
if stream:
|
if stream:
|
||||||
|
|
@ -208,7 +212,9 @@ class BaseChatStep(IChatStep):
|
||||||
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
|
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
|
||||||
if os.path.exists(code_path):
|
if os.path.exists(code_path):
|
||||||
os.remove(code_path)
|
os.remove(code_path)
|
||||||
|
from system_manage.mcp_agent import call_records
|
||||||
return {
|
return {
|
||||||
|
'engine_calls': call_records(self.context.get('engine_body', {})),
|
||||||
'step_type': 'chat_step',
|
'step_type': 'chat_step',
|
||||||
'run_time': self.context['run_time'],
|
'run_time': self.context['run_time'],
|
||||||
'model_id': str(manage.context['model_id']),
|
'model_id': str(manage.context['model_id']),
|
||||||
|
|
@ -231,25 +237,15 @@ class BaseChatStep(IChatStep):
|
||||||
|
|
||||||
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_ids, tool_ids,
|
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_ids, tool_ids,
|
||||||
mcp_output_enable, chat_model, message_list):
|
mcp_output_enable, chat_model, message_list):
|
||||||
if not mcp_enable and not tool_enable:
|
if not mcp_enable and not tool_enable and not self.context.get('engine_tools'):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
mcp_servers_config = {}
|
mcp_servers_config = {}
|
||||||
|
|
||||||
# 迁移过来mcp_source是None
|
from system_manage.mcp_agent import context_for
|
||||||
if mcp_source is None:
|
engine_context = context_for(self.context['engine_body'], engine_tools=self.context.get('engine_tools'),
|
||||||
mcp_source = 'custom'
|
tool_ids=mcp_tool_ids if mcp_enable and mcp_source not in (None, 'custom') else [],
|
||||||
if mcp_enable:
|
custom=mcp_servers if mcp_enable and mcp_source in (None, 'custom') else None)
|
||||||
# 兼容老数据
|
|
||||||
if not mcp_tool_ids:
|
|
||||||
mcp_tool_ids = []
|
|
||||||
if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers:
|
|
||||||
mcp_servers_config = json.loads(mcp_servers)
|
|
||||||
elif mcp_tool_ids:
|
|
||||||
mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values()
|
|
||||||
for mcp_tool in mcp_tools:
|
|
||||||
if mcp_tool and mcp_tool['is_active']:
|
|
||||||
mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])}
|
|
||||||
|
|
||||||
if tool_enable:
|
if tool_enable:
|
||||||
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
|
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
|
||||||
|
|
@ -269,8 +265,8 @@ class BaseChatStep(IChatStep):
|
||||||
self.context['execute_ids'].append(_id)
|
self.context['execute_ids'].append(_id)
|
||||||
mcp_servers_config[str(tool.id)] = tool_config
|
mcp_servers_config[str(tool.id)] = tool_config
|
||||||
|
|
||||||
if len(mcp_servers_config) > 0:
|
if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or engine_context['engine_tools']:
|
||||||
return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable)
|
return mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -332,6 +328,7 @@ class BaseChatStep(IChatStep):
|
||||||
no_references_setting, problem_text, mcp_enable, mcp_tool_ids, mcp_servers, mcp_source, tool_enable, tool_ids,
|
no_references_setting, problem_text, mcp_enable, mcp_tool_ids, mcp_servers, mcp_source, tool_enable, tool_ids,
|
||||||
mcp_output_enable)
|
mcp_output_enable)
|
||||||
chat_record_id = uuid.uuid7()
|
chat_record_id = uuid.uuid7()
|
||||||
|
self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id)
|
||||||
r = StreamingHttpResponse(
|
r = StreamingHttpResponse(
|
||||||
streaming_content=event_content(chat_result, chat_id, chat_record_id, paragraph_list,
|
streaming_content=event_content(chat_result, chat_id, chat_record_id, paragraph_list,
|
||||||
post_response_handler, manage, self, chat_model, message_list, problem_text,
|
post_response_handler, manage, self, chat_model, message_list, problem_text,
|
||||||
|
|
@ -375,7 +372,8 @@ class BaseChatStep(IChatStep):
|
||||||
chat_model, message_list,
|
chat_model, message_list,
|
||||||
)
|
)
|
||||||
if mcp_result:
|
if mcp_result:
|
||||||
return mcp_result, True
|
from application.flow.tools import collect_mcp_response
|
||||||
|
return collect_mcp_response(mcp_result), True
|
||||||
return chat_model.invoke(message_list), True
|
return chat_model.invoke(message_list), True
|
||||||
|
|
||||||
def execute_block(self, message_list: List[BaseMessage],
|
def execute_block(self, message_list: List[BaseMessage],
|
||||||
|
|
@ -401,6 +399,7 @@ class BaseChatStep(IChatStep):
|
||||||
reasoning = Reasoning(reasoning_content_start,
|
reasoning = Reasoning(reasoning_content_start,
|
||||||
reasoning_content_end)
|
reasoning_content_end)
|
||||||
chat_record_id = uuid.uuid7()
|
chat_record_id = uuid.uuid7()
|
||||||
|
self.context.get('engine_body', {})['chat_record_id'] = str(chat_record_id)
|
||||||
# 调用模型
|
# 调用模型
|
||||||
try:
|
try:
|
||||||
chat_result, is_ai_chat = self.get_block_result(message_list, chat_model, paragraph_list,
|
chat_result, is_ai_chat = self.get_block_result(message_list, chat_model, paragraph_list,
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
"""
|
"""
|
||||||
@project: maxkb
|
@project: maxkb
|
||||||
|
|
@ -60,14 +61,19 @@ class BaseSearchDatasetStep(ISearchDatasetStep):
|
||||||
knowledge_id_list)
|
knowledge_id_list)
|
||||||
if len(knowledge_id_list) == 0:
|
if len(knowledge_id_list) == 0:
|
||||||
return []
|
return []
|
||||||
|
from knowledge.versions import resolve_search
|
||||||
|
knowledge_id_list = resolve_search(knowledge_id_list, manage.context.get('knowledge_version_map'), workspace_id)
|
||||||
|
# Exclusion IDs must come from the selected content, including frozen inactive documents.
|
||||||
|
from knowledge.models import Document
|
||||||
|
exclude_document_id_list = list(Document.objects.filter(knowledge_id__in=knowledge_id_list,
|
||||||
|
is_active=False).values_list('id', flat=True))
|
||||||
exec_problem_text = padding_problem_text if padding_problem_text is not None else problem_text
|
exec_problem_text = padding_problem_text if padding_problem_text is not None else problem_text
|
||||||
model_id = get_embedding_id(knowledge_id_list)
|
model_id = get_embedding_id(knowledge_id_list)
|
||||||
model = get_model_by_id(model_id, workspace_id)
|
model = get_model_by_id(model_id, workspace_id)
|
||||||
if model.model_type != "EMBEDDING":
|
if model.model_type != "EMBEDDING":
|
||||||
raise Exception(_("Model does not exist"))
|
raise Exception(_("Model does not exist"))
|
||||||
self.context['model_name'] = model.name
|
self.context['model_name'] = model.name
|
||||||
default_params = get_model_default_params(model)
|
embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list)
|
||||||
embedding_model = ModelManage.get_model(model_id, lambda _id: get_model(model, **{**default_params}))
|
|
||||||
embedding_value = embedding_model.embed_query(exec_problem_text)
|
embedding_value = embedding_model.embed_query(exec_problem_text)
|
||||||
vector = VectorStore.get_embedding_vector()
|
vector = VectorStore.get_embedding_vector()
|
||||||
embedding_list = vector.query(exec_problem_text, embedding_value, knowledge_id_list, None, exclude_document_id_list,
|
embedding_list = vector.query(exec_problem_text, embedding_value, knowledge_id_list, None, exclude_document_id_list,
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ from .loop_continue_node import BaseLoopContinueNode
|
||||||
from .loop_node import *
|
from .loop_node import *
|
||||||
from .loop_start_node import *
|
from .loop_start_node import *
|
||||||
from .mcp_node import BaseMcpNode
|
from .mcp_node import BaseMcpNode
|
||||||
|
from .component_node import BaseComponentNode
|
||||||
from .parameter_extraction_node import BaseParameterExtractionNode
|
from .parameter_extraction_node import BaseParameterExtractionNode
|
||||||
from .question_node import *
|
from .question_node import *
|
||||||
from .reranker_node import *
|
from .reranker_node import *
|
||||||
|
|
@ -42,7 +43,7 @@ node_list = [BaseStartStepNode, BaseChatNode, BaseSearchKnowledgeNode, BaseSearc
|
||||||
BaseToolNodeNode, BaseToolLibNodeNode, BaseRerankerNode, BaseApplicationNode,
|
BaseToolNodeNode, BaseToolLibNodeNode, BaseRerankerNode, BaseApplicationNode,
|
||||||
BaseDocumentExtractNode,
|
BaseDocumentExtractNode,
|
||||||
BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,
|
BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode,
|
||||||
BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseTextToVideoNode, BaseImageToVideoNode,
|
BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseComponentNode, BaseTextToVideoNode, BaseImageToVideoNode,
|
||||||
BaseVideoUnderstandNode,
|
BaseVideoUnderstandNode,
|
||||||
BaseIntentNode, BaseLoopNode, BaseLoopStartStepNode,
|
BaseIntentNode, BaseLoopNode, BaseLoopStartStepNode,
|
||||||
BaseLoopContinueNode,
|
BaseLoopContinueNode,
|
||||||
|
|
|
||||||
|
|
@ -206,29 +206,23 @@ class BaseChatNode(IChatNode):
|
||||||
|
|
||||||
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_id, mcp_tool_ids, tool_ids,
|
def _handle_mcp_request(self, mcp_enable, tool_enable, mcp_source, mcp_servers, mcp_tool_id, mcp_tool_ids, tool_ids,
|
||||||
mcp_output_enable, chat_model, message_list, history_message, question):
|
mcp_output_enable, chat_model, message_list, history_message, question):
|
||||||
if not mcp_enable and not tool_enable:
|
enabled = (self.node_params.get('model_setting') or {}).get('engine_tools', [])
|
||||||
|
if not mcp_enable and not tool_enable and not enabled:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
mcp_servers_config = {}
|
mcp_servers_config = {}
|
||||||
|
|
||||||
# 迁移过来mcp_source是None
|
from system_manage.mcp_agent import context_for
|
||||||
if mcp_source is None:
|
ids = list(mcp_tool_ids or [])
|
||||||
mcp_source = 'custom'
|
if mcp_tool_id: ids.append(mcp_tool_id)
|
||||||
if mcp_enable:
|
body = dict(self.workflow_manage.get_body())
|
||||||
# 兼容老数据
|
# Explicit AI-node selection is limited to knowledge bases associated with this workflow.
|
||||||
if not mcp_tool_ids:
|
from system_manage.mcp_workflows import knowledge_scope
|
||||||
mcp_tool_ids = []
|
body['knowledge_id_list'] = knowledge_scope(self.workflow_manage.flow)
|
||||||
if mcp_tool_id:
|
engine_context = context_for(body, engine_tools=enabled,
|
||||||
mcp_tool_ids = list(set(mcp_tool_ids + [mcp_tool_id]))
|
tool_ids=ids if mcp_enable and mcp_source not in (None, 'custom') else [],
|
||||||
if mcp_source == 'custom' and mcp_servers is not None and '"stdio"' not in mcp_servers:
|
custom=self.handle_variables(json.loads(mcp_servers) if isinstance(mcp_servers, str) else mcp_servers) if mcp_enable and mcp_source in (None, 'custom') and mcp_servers else None,
|
||||||
mcp_servers_config = json.loads(mcp_servers)
|
transform=self.handle_variables, node_id=self.id)
|
||||||
mcp_servers_config = self.handle_variables(mcp_servers_config)
|
|
||||||
elif mcp_tool_ids:
|
|
||||||
mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values()
|
|
||||||
for mcp_tool in mcp_tools:
|
|
||||||
if mcp_tool and mcp_tool['is_active']:
|
|
||||||
mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool['code'])}
|
|
||||||
mcp_servers_config = self.handle_variables(mcp_servers_config)
|
|
||||||
|
|
||||||
if tool_enable:
|
if tool_enable:
|
||||||
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
|
if tool_ids and len(tool_ids) > 0: # 如果有工具ID,则将其转换为MCP
|
||||||
|
|
@ -248,8 +242,8 @@ class BaseChatNode(IChatNode):
|
||||||
self.context['execute_ids'].append(_id)
|
self.context['execute_ids'].append(_id)
|
||||||
mcp_servers_config[str(tool.id)] = tool_config
|
mcp_servers_config[str(tool.id)] = tool_config
|
||||||
|
|
||||||
if len(mcp_servers_config) > 0:
|
if mcp_servers_config or engine_context['tool_ids'] or engine_context['custom'] or enabled:
|
||||||
r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable)
|
r = mcp_response_generator(chat_model, message_list, json.dumps(mcp_servers_config), mcp_output_enable, engine_context)
|
||||||
return NodeResult(
|
return NodeResult(
|
||||||
{'result': r, 'chat_model': chat_model, 'message_list': message_list,
|
{'result': r, 'chat_model': chat_model, 'message_list': message_list,
|
||||||
'history_message': [{'content': message.content, 'role': message.type} for message in
|
'history_message': [{'content': message.content, 'role': message.type} for message in
|
||||||
|
|
@ -315,7 +309,9 @@ class BaseChatNode(IChatNode):
|
||||||
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
|
code_path = f'{executor.sandbox_path}/execute/{tool_id}.py'
|
||||||
if os.path.exists(code_path):
|
if os.path.exists(code_path):
|
||||||
os.remove(code_path)
|
os.remove(code_path)
|
||||||
|
from system_manage.mcp_agent import call_records
|
||||||
return {
|
return {
|
||||||
|
'engine_calls': call_records(self.workflow_manage.get_body(), self.id),
|
||||||
'name': self.node.properties.get('stepName'),
|
'name': self.node.properties.get('stepName'),
|
||||||
"index": index,
|
"index": index,
|
||||||
'run_time': self.context.get('run_time'),
|
'run_time': self.context.get('run_time'),
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
from .base_component_node import BaseComponentNode
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
from copy import deepcopy
|
||||||
|
from rest_framework import serializers
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
|
from application.flow.i_step_node import INode, NodeResult
|
||||||
|
from system_manage import mcp_runtime
|
||||||
|
|
||||||
|
|
||||||
|
class ComponentParams(serializers.Serializer):
|
||||||
|
component_id = serializers.CharField()
|
||||||
|
parameters = serializers.DictField(default=dict)
|
||||||
|
sources = serializers.DictField(default=dict)
|
||||||
|
timeout = serializers.IntegerField(default=60, min_value=1, max_value=600)
|
||||||
|
retries = serializers.IntegerField(default=0, min_value=0, max_value=2)
|
||||||
|
priority = serializers.IntegerField(default=5, min_value=0, max_value=9)
|
||||||
|
|
||||||
|
|
||||||
|
def workflow_identity(workflow):
|
||||||
|
from application.models import Application
|
||||||
|
from users.models import User
|
||||||
|
body = workflow.get_body()
|
||||||
|
app = Application.objects.filter(id=body['application_id'], workspace_id=body['workspace_id']).first()
|
||||||
|
if not app or not app.user_id:
|
||||||
|
raise ValidationError('应用不存在或没有可用的所有者')
|
||||||
|
user = User.objects.get(id=body.get('engine_user_id') or app.user_id, is_active=True)
|
||||||
|
return body, user
|
||||||
|
|
||||||
|
|
||||||
|
class BaseComponentNode(INode):
|
||||||
|
type = 'component-node'
|
||||||
|
|
||||||
|
def get_node_params_serializer_class(self):
|
||||||
|
return ComponentParams
|
||||||
|
|
||||||
|
def _run(self):
|
||||||
|
return self.execute(**self.node_params_serializer.data)
|
||||||
|
|
||||||
|
def execute(self, component_id, parameters=None, sources=None, timeout=60, retries=0, priority=5, **kwargs):
|
||||||
|
body, user = workflow_identity(self.workflow_manage)
|
||||||
|
def render(value):
|
||||||
|
if isinstance(value, str): return self.workflow_manage.generate_prompt(value)
|
||||||
|
if isinstance(value, dict): return {key: render(item) for key, item in value.items()}
|
||||||
|
if isinstance(value, list): return [render(item) for item in value]
|
||||||
|
return value
|
||||||
|
params = deepcopy(parameters or {})
|
||||||
|
for key, value in params.items():
|
||||||
|
if (sources or {}).get(key) == 'reference':
|
||||||
|
if not isinstance(value, list) or len(value) < 2:
|
||||||
|
raise ValidationError('请选择有效的上游变量')
|
||||||
|
params[key] = self.workflow_manage.get_reference_field(value[0], value[1:])
|
||||||
|
else:
|
||||||
|
params[key] = render(value)
|
||||||
|
result = mcp_runtime.invoke(body['workspace_id'], user, component_id, params,
|
||||||
|
parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'),
|
||||||
|
scope={**body, 'node_id': self.id, 'source': 'workflow-component'},
|
||||||
|
timeout=timeout, retries=retries, priority=priority)
|
||||||
|
return NodeResult(result, {})
|
||||||
|
|
||||||
|
def save_context(self, details, workflow_manage):
|
||||||
|
self.context.update({key: details.get(key) for key in ('text', 'structured', 'result', 'task_id', 'component_id')})
|
||||||
|
|
||||||
|
def get_details(self, index, **kwargs):
|
||||||
|
return {'name': self.node.properties.get('stepName'), 'index': index, 'type': self.type,
|
||||||
|
'status': self.status, 'err_message': self.err_message, 'run_time': self.context.get('run_time'),
|
||||||
|
**{key: self.context.get(key) for key in ('component_id', 'text', 'structured', 'task_id')}}
|
||||||
|
|
@ -9,12 +9,16 @@ from application.flow.i_step_node import INode, NodeResult
|
||||||
|
|
||||||
|
|
||||||
class McpNodeSerializer(serializers.Serializer):
|
class McpNodeSerializer(serializers.Serializer):
|
||||||
mcp_servers = serializers.JSONField(required=True, label=_("Mcp servers"))
|
mcp_servers = serializers.JSONField(required=False, default=dict, label=_("Mcp servers"))
|
||||||
mcp_server = serializers.CharField(required=True, label=_("Mcp server"))
|
mcp_server = serializers.CharField(required=True, label=_("Mcp server"))
|
||||||
mcp_tool = serializers.CharField(required=True, label=_("Mcp tool"))
|
mcp_tool = serializers.CharField(required=True, label=_("Mcp tool"))
|
||||||
mcp_tool_id = serializers.CharField(required=False, label=_("Mcp tool"), allow_null=True, allow_blank=True)
|
mcp_tool_id = serializers.CharField(required=False, label=_("Mcp tool"), allow_null=True, allow_blank=True)
|
||||||
mcp_source = serializers.CharField(required=False, label=_("Mcp source"), allow_blank=True, allow_null=True)
|
mcp_source = serializers.CharField(required=False, label=_("Mcp source"), allow_blank=True, allow_null=True)
|
||||||
tool_params = serializers.DictField(required=True, label=_("Tool parameters"))
|
tool_params = serializers.DictField(required=True, label=_("Tool parameters"))
|
||||||
|
tool_form_field = serializers.ListField(required=False, default=list)
|
||||||
|
timeout = serializers.IntegerField(default=60, min_value=1, max_value=600)
|
||||||
|
retries = serializers.IntegerField(default=0, min_value=0, max_value=2)
|
||||||
|
priority = serializers.IntegerField(default=5, min_value=0, max_value=9)
|
||||||
|
|
||||||
|
|
||||||
class IMcpNode(INode):
|
class IMcpNode(INode):
|
||||||
|
|
|
||||||
|
|
@ -1,75 +1,59 @@
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
import asyncio
|
|
||||||
import json
|
import json
|
||||||
|
from copy import deepcopy
|
||||||
from typing import List
|
from typing import List
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
from django.db.models import QuerySet
|
|
||||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
|
||||||
|
|
||||||
from application.flow.i_step_node import NodeResult
|
from application.flow.i_step_node import NodeResult
|
||||||
from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode
|
from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode
|
||||||
from tools.models import Tool
|
from application.flow.step_node.component_node.base_component_node import workflow_identity
|
||||||
|
from system_manage import mcp_runtime
|
||||||
|
|
||||||
|
|
||||||
class BaseMcpNode(IMcpNode):
|
class BaseMcpNode(IMcpNode):
|
||||||
def save_context(self, details, workflow_manage):
|
def save_context(self, details, workflow_manage):
|
||||||
self.context['result'] = details.get('result')
|
self.context.update({key: details.get(key) for key in ('result', 'text', 'structured', 'tool_params', 'mcp_tool', 'mcp_server', 'attempts', 'task_id')})
|
||||||
self.context['tool_params'] = details.get('tool_params')
|
|
||||||
self.context['mcp_tool'] = details.get('mcp_tool')
|
|
||||||
|
|
||||||
def execute(self, mcp_servers, mcp_server, mcp_tool, mcp_tool_id, mcp_source, tool_params,**kwargs) -> NodeResult:
|
def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, mcp_tool_id=None, mcp_source=None,
|
||||||
if mcp_source == 'referencing':
|
tool_form_field=None, timeout=60, retries=0, priority=5, **kwargs):
|
||||||
if not mcp_tool_id:
|
body, user = workflow_identity(self.workflow_manage)
|
||||||
raise ValueError("MCP tool ID is required when mcp_source is 'referencing'.")
|
params = self.handle_variables(deepcopy(tool_params), tool_form_field or [])
|
||||||
tool = QuerySet(Tool).filter(id=mcp_tool_id).first()
|
key = 'mcp:' + str(mcp_tool_id) if mcp_source == 'referencing' else 'mcp-custom'
|
||||||
if not tool:
|
result = mcp_runtime.invoke(body['workspace_id'], user, key,
|
||||||
raise ValueError(f"Tool with ID {mcp_tool_id} not found.")
|
{'servers': self.handle_variables(mcp_runtime.connections(mcp_servers)) if key == 'mcp-custom' else None,
|
||||||
if not tool.is_active:
|
'server': mcp_server, 'tool': mcp_tool, 'arguments': params},
|
||||||
raise ValueError(f"Tool with ID {mcp_tool_id} is inactive.")
|
parent=body.get('engine_task_id'), application_id=body['application_id'], chat_id=body.get('chat_id'),
|
||||||
servers = json.loads(tool.code)
|
scope={**body, 'node_id': self.id, 'source': 'workflow-mcp'}, timeout=timeout, retries=retries, priority=priority)
|
||||||
servers = self.handle_variables(servers) # 处理servers中的变量
|
return NodeResult({**result, 'tool_params': params}, {})
|
||||||
params = json.loads(json.dumps(tool_params))
|
|
||||||
params = self.handle_variables(params)
|
def handle_variables(self, value, fields=None):
|
||||||
|
if isinstance(value, dict):
|
||||||
|
output = {}
|
||||||
|
for key, item in value.items():
|
||||||
|
field = next((f for f in fields or [] if f.get('field') == key), {})
|
||||||
|
if field.get('source') == 'custom' and field.get('input_type') == 'JsonInput' and isinstance(item, str):
|
||||||
|
try: output[key] = json.loads(self.workflow_manage.generate_prompt(item))
|
||||||
|
except ValueError: raise ValidationError('工具参数必须为有效 JSON') from None
|
||||||
|
elif field.get('source') == 'custom':
|
||||||
|
output[key] = self.workflow_manage.generate_prompt(item) if isinstance(item, str) else item
|
||||||
|
elif field.get('source') == 'referencing':
|
||||||
|
if not isinstance(item, list) or len(item) < 2: raise ValidationError('请选择工具参数的上游变量')
|
||||||
|
output[key] = self.get_reference_content(item)
|
||||||
else:
|
else:
|
||||||
servers = json.loads(mcp_servers)
|
output[key] = self.handle_variables(item, fields)
|
||||||
servers = self.handle_variables(servers) # 处理servers中的变量
|
return output
|
||||||
params = json.loads(json.dumps(tool_params))
|
if isinstance(value, str): return self.workflow_manage.generate_prompt(value)
|
||||||
params = self.handle_variables(params)
|
if isinstance(value, list):
|
||||||
|
known = set(self.workflow_manage.flow.node_map) | {'global', 'chat', 'loop'}
|
||||||
async def call_tool(t, a):
|
if len(value) >= 2 and all(isinstance(v, str) for v in value) and value[0] in known:
|
||||||
client = MultiServerMCPClient(servers)
|
return self.get_reference_content(value)
|
||||||
async with client.session(mcp_server) as s:
|
return [self.handle_variables(item, fields) for item in value]
|
||||||
return await s.call_tool(t, a)
|
return value
|
||||||
|
|
||||||
res = asyncio.run(call_tool(mcp_tool, params))
|
|
||||||
return NodeResult(
|
|
||||||
{'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {})
|
|
||||||
|
|
||||||
def handle_variables(self, tool_params):
|
|
||||||
# 处理参数中的变量
|
|
||||||
for k, v in tool_params.items():
|
|
||||||
if type(v) == str:
|
|
||||||
tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k])
|
|
||||||
if type(v) == dict:
|
|
||||||
self.handle_variables(v)
|
|
||||||
if (type(v) == list) and (type(v[0]) == str):
|
|
||||||
tool_params[k] = self.get_reference_content(v)
|
|
||||||
return tool_params
|
|
||||||
|
|
||||||
def get_reference_content(self, fields: List[str]):
|
def get_reference_content(self, fields: List[str]):
|
||||||
return self.workflow_manage.get_reference_field(
|
return self.workflow_manage.get_reference_field(fields[0], fields[1:])
|
||||||
fields[0],
|
|
||||||
fields[1:])
|
|
||||||
|
|
||||||
def get_details(self, index: int, **kwargs):
|
def get_details(self, index: int, **kwargs):
|
||||||
return {
|
return {'name': self.node.properties.get('stepName'), 'index': index,
|
||||||
'name': self.node.properties.get('stepName'),
|
'run_time': self.context.get('run_time'), 'status': self.status,
|
||||||
"index": index,
|
'err_message': self.err_message, 'type': self.node.type,
|
||||||
'run_time': self.context.get('run_time'),
|
**{key: self.context.get(key) for key in ('mcp_tool', 'mcp_server', 'tool_params', 'result', 'text', 'structured', 'attempts', 'task_id')}}
|
||||||
'status': self.status,
|
|
||||||
'err_message': self.err_message,
|
|
||||||
'type': self.node.type,
|
|
||||||
'mcp_tool': self.context.get('mcp_tool'),
|
|
||||||
'tool_params': self.context.get('tool_params'),
|
|
||||||
'result': self.context.get('result'),
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ from common.utils.common import flat_map
|
||||||
|
|
||||||
|
|
||||||
class DatasetSettingSerializer(serializers.Serializer):
|
class DatasetSettingSerializer(serializers.Serializer):
|
||||||
|
version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False)
|
||||||
# 需要查询的条数
|
# 需要查询的条数
|
||||||
top_n = serializers.IntegerField(required=True,
|
top_n = serializers.IntegerField(required=True,
|
||||||
label=_("Reference segment number"))
|
label=_("Reference segment number"))
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
|
||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
"""
|
"""
|
||||||
@project: maxkb
|
@project: maxkb
|
||||||
|
|
@ -100,9 +101,17 @@ class BaseSearchKnowledgeNode(ISearchKnowledgeStepNode):
|
||||||
knowledge_id_list)
|
knowledge_id_list)
|
||||||
if len(knowledge_id_list) == 0:
|
if len(knowledge_id_list) == 0:
|
||||||
return get_none_result(question)
|
return get_none_result(question)
|
||||||
|
from knowledge.versions import resolve_search, resolve_document_scope
|
||||||
|
knowledge_id_list = resolve_search(knowledge_id_list,
|
||||||
|
self.workflow_manage.get_body().get('knowledge_version_map'),
|
||||||
|
self.workflow_manage.get_body().get('workspace_id'))
|
||||||
|
document_id_list = resolve_document_scope(document_id_list,
|
||||||
|
self.workflow_manage.get_body().get('knowledge_version_map'),
|
||||||
|
self.workflow_manage.get_body().get('workspace_id'))
|
||||||
model_id = get_embedding_id(knowledge_id_list)
|
model_id = get_embedding_id(knowledge_id_list)
|
||||||
workspace_id = self.workflow_manage.get_body().get('workspace_id')
|
workspace_id = self.workflow_manage.get_body().get('workspace_id')
|
||||||
embedding_model = get_model_instance_by_model_workspace_id(model_id, workspace_id)
|
get_model_instance_by_model_workspace_id(model_id, workspace_id) # Preserve resource access validation.
|
||||||
|
embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list)
|
||||||
embedding_value = embedding_model.embed_query(question)
|
embedding_value = embedding_model.embed_query(question)
|
||||||
vector = VectorStore.get_embedding_vector()
|
vector = VectorStore.get_embedding_vector()
|
||||||
exclude_document_id_list = [str(document.id) for document in
|
exclude_document_id_list = [str(document.id) for document in
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ import threading
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
from django.http import StreamingHttpResponse
|
from django.http import StreamingHttpResponse
|
||||||
from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk
|
from langchain_core.messages import BaseMessageChunk, BaseMessage, ToolMessage, AIMessageChunk, AIMessage, SystemMessage
|
||||||
from langchain_mcp_adapters.client import MultiServerMCPClient
|
from langchain_mcp_adapters.client import MultiServerMCPClient
|
||||||
from langgraph.prebuilt import create_react_agent
|
from langgraph.prebuilt import create_react_agent
|
||||||
from application.flow.i_step_node import WorkFlowPostHandler
|
from application.flow.i_step_node import WorkFlowPostHandler
|
||||||
|
|
@ -267,17 +267,23 @@ def get_global_loop():
|
||||||
return _global_loop
|
return _global_loop
|
||||||
|
|
||||||
|
|
||||||
async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True):
|
async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None):
|
||||||
client = MultiServerMCPClient(json.loads(mcp_servers))
|
servers = json.loads(mcp_servers)
|
||||||
tools = await client.get_tools()
|
tools = await MultiServerMCPClient(servers).get_tools() if servers else []
|
||||||
|
if engine_context:
|
||||||
|
from system_manage.mcp_agent import tools as engine_tools
|
||||||
|
tools += await engine_tools(engine_context)
|
||||||
agent = create_react_agent(chat_model, tools)
|
agent = create_react_agent(chat_model, tools)
|
||||||
response = agent.astream({"messages": message_list}, stream_mode='messages')
|
messages = message_list
|
||||||
|
if not any(isinstance(message, SystemMessage) for message in message_list):
|
||||||
|
messages = [SystemMessage(content='你是应用助手。根据用户请求选择已授权工具。需要外部数据或生成图谱时实际调用工具,不要编造工具结果。工具执行完成后依据结果回答,保存导图后返回打开链接,避免重复调用。'), *message_list]
|
||||||
|
response = agent.astream({"messages": messages}, stream_mode='messages')
|
||||||
|
|
||||||
# 用于存储工具调用信息
|
# 用于存储工具调用信息
|
||||||
tool_calls_info = {}
|
tool_calls_info = {}
|
||||||
|
|
||||||
async for chunk in response:
|
async for chunk in response:
|
||||||
if isinstance(chunk[0], AIMessageChunk):
|
if isinstance(chunk[0], (AIMessageChunk, AIMessage)):
|
||||||
tool_calls = chunk[0].additional_kwargs.get('tool_calls', [])
|
tool_calls = chunk[0].additional_kwargs.get('tool_calls', [])
|
||||||
for tool_call in tool_calls:
|
for tool_call in tool_calls:
|
||||||
tool_id = tool_call.get('id', '')
|
tool_id = tool_call.get('id', '')
|
||||||
|
|
@ -287,6 +293,10 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
|
||||||
'name': tool_call.get('function', {}).get('name', ''),
|
'name': tool_call.get('function', {}).get('name', ''),
|
||||||
'input': tool_call.get('function', {}).get('arguments', '')
|
'input': tool_call.get('function', {}).get('arguments', '')
|
||||||
}
|
}
|
||||||
|
for tool_call in chunk[0].tool_calls:
|
||||||
|
if tool_call.get('id'):
|
||||||
|
tool_calls_info[tool_call['id']] = {'name': tool_call['name'],
|
||||||
|
'input': json.dumps(tool_call.get('args', {}), ensure_ascii=False)}
|
||||||
yield chunk[0]
|
yield chunk[0]
|
||||||
|
|
||||||
if mcp_output_enable and isinstance(chunk[0], ToolMessage):
|
if mcp_output_enable and isinstance(chunk[0], ToolMessage):
|
||||||
|
|
@ -303,14 +313,30 @@ async def _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_
|
||||||
yield chunk[0]
|
yield chunk[0]
|
||||||
|
|
||||||
|
|
||||||
def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True):
|
def collect_mcp_response(response):
|
||||||
|
"""Use the same tool stream for the non-streaming chat API."""
|
||||||
|
from langchain_core.messages import AIMessage
|
||||||
|
parts, reasoning, metadata = [], [], {}
|
||||||
|
for chunk in response:
|
||||||
|
if isinstance(chunk.content, str):
|
||||||
|
parts.append(chunk.content)
|
||||||
|
else:
|
||||||
|
parts.extend(item.get('text', '') for item in chunk.content if isinstance(item, dict))
|
||||||
|
metadata.update(chunk.response_metadata)
|
||||||
|
value = chunk.additional_kwargs.get('reasoning_content')
|
||||||
|
if value: reasoning.append(value)
|
||||||
|
if reasoning: metadata['reasoning_content'] = ''.join(reasoning)
|
||||||
|
return AIMessage(content=''.join(parts), response_metadata=metadata)
|
||||||
|
|
||||||
|
|
||||||
|
def mcp_response_generator(chat_model, message_list, mcp_servers, mcp_output_enable=True, engine_context=None):
|
||||||
"""使用全局事件循环,不创建新实例"""
|
"""使用全局事件循环,不创建新实例"""
|
||||||
result_queue = queue.Queue()
|
result_queue = queue.Queue()
|
||||||
loop = get_global_loop() # 使用共享循环
|
loop = get_global_loop() # 使用共享循环
|
||||||
|
|
||||||
async def _run():
|
async def _run():
|
||||||
try:
|
try:
|
||||||
async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable)
|
async_gen = _yield_mcp_response(chat_model, message_list, mcp_servers, mcp_output_enable, engine_context)
|
||||||
async for chunk in async_gen:
|
async for chunk in async_gen:
|
||||||
result_queue.put(('data', chunk))
|
result_queue.put(('data', chunk))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -98,6 +98,7 @@ class NoReferencesSetting(serializers.Serializer):
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeSettingSerializer(serializers.Serializer):
|
class KnowledgeSettingSerializer(serializers.Serializer):
|
||||||
|
version_selection = serializers.DictField(child=serializers.CharField(allow_blank=True), required=False)
|
||||||
top_n = serializers.FloatField(required=True, max_value=10000, min_value=1,
|
top_n = serializers.FloatField(required=True, max_value=10000, min_value=1,
|
||||||
label=_("Reference segment number"))
|
label=_("Reference segment number"))
|
||||||
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
|
similarity = serializers.FloatField(required=True, max_value=1, min_value=0,
|
||||||
|
|
@ -132,6 +133,8 @@ class ModelKnowledgeAssociation(serializers.Serializer):
|
||||||
if not QuerySet(Model).filter(id=model_id).exists():
|
if not QuerySet(Model).filter(id=model_id).exists():
|
||||||
raise AppApiException(500, f'{_("Model does not exist")}【{model_id}】')
|
raise AppApiException(500, f'{_("Model does not exist")}【{model_id}】')
|
||||||
knowledge_id_list = list(set(self.data.get('knowledge_id_list', [])))
|
knowledge_id_list = list(set(self.data.get('knowledge_id_list', [])))
|
||||||
|
if Knowledge.objects.filter(id__in=knowledge_id_list, meta__has_key='_version_snapshot').exists():
|
||||||
|
raise AppApiException(500, '请关联原知识库,再选择历史版本')
|
||||||
if workspace_id is not None and len(workspace_id) > 0:
|
if workspace_id is not None and len(workspace_id) > 0:
|
||||||
exist_knowledge_id_list = [
|
exist_knowledge_id_list = [
|
||||||
str(knowledge.get('id')) for knowledge in KnowledgeSerializer.Query(data={
|
str(knowledge.get('id')) for knowledge in KnowledgeSerializer.Query(data={
|
||||||
|
|
@ -148,6 +151,8 @@ class ModelKnowledgeAssociation(serializers.Serializer):
|
||||||
|
|
||||||
|
|
||||||
class ModelSettingSerializer(serializers.Serializer):
|
class ModelSettingSerializer(serializers.Serializer):
|
||||||
|
engine_tools = serializers.ListField(required=False, child=serializers.ChoiceField(
|
||||||
|
choices=['knowledge-search', 'mindmap-save']), max_length=2)
|
||||||
prompt = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
prompt = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
||||||
label=_("Prompt word"))
|
label=_("Prompt word"))
|
||||||
system = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
system = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
|
||||||
|
|
@ -272,6 +277,9 @@ class ApplicationCreateSerializer(serializers.Serializer):
|
||||||
ModelKnowledgeAssociation(data={'user_id': user_id, 'workspace_id': workspace_id,
|
ModelKnowledgeAssociation(data={'user_id': user_id, 'workspace_id': workspace_id,
|
||||||
'model_id': self.data.get('model_id'),
|
'model_id': self.data.get('model_id'),
|
||||||
'knowledge_id_list': self.data.get('knowledge_id_list')}).is_valid()
|
'knowledge_id_list': self.data.get('knowledge_id_list')}).is_valid()
|
||||||
|
from knowledge.versions import validate_selection
|
||||||
|
validate_selection((self.data.get('knowledge_setting') or {}).get('version_selection', {}),
|
||||||
|
workspace_id, self.data.get('knowledge_id_list') or [])
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def to_application_model(user_id: str, workspace_id: str, application: Dict):
|
def to_application_model(user_id: str, workspace_id: str, application: Dict):
|
||||||
|
|
@ -671,21 +679,8 @@ class ApplicationOperateSerializer(serializers.Serializer):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
McpServersSerializer(data=instance).is_valid(raise_exception=True)
|
McpServersSerializer(data=instance).is_valid(raise_exception=True)
|
||||||
servers = json.loads(instance.get('mcp_servers'))
|
from system_manage.mcp_runtime import discover
|
||||||
for server, config in servers.items():
|
return discover(instance.get('mcp_servers'))
|
||||||
if config.get('transport') not in ['sse', 'streamable_http']:
|
|
||||||
raise AppApiException(500, _('Only support transport=sse or transport=streamable_http'))
|
|
||||||
tools = []
|
|
||||||
for server in servers:
|
|
||||||
tools += [
|
|
||||||
{
|
|
||||||
'server': server,
|
|
||||||
'name': tool.name,
|
|
||||||
'description': tool.description,
|
|
||||||
'args_schema': tool.args_schema,
|
|
||||||
}
|
|
||||||
for tool in asyncio.run(get_mcp_tools({server: servers[server]}))]
|
|
||||||
return tools
|
|
||||||
|
|
||||||
def delete(self, with_valid=True):
|
def delete(self, with_valid=True):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
|
|
@ -780,6 +775,13 @@ class ApplicationOperateSerializer(serializers.Serializer):
|
||||||
publish_user_name=user.username,
|
publish_user_name=user.username,
|
||||||
workspace_id=workspace_id)
|
workspace_id=workspace_id)
|
||||||
self.reset_application_version(work_flow_version, application)
|
self.reset_application_version(work_flow_version, application)
|
||||||
|
from knowledge.versions import application_selection, validate_selection
|
||||||
|
root_ids = [str(k) for k in ApplicationKnowledgeMapping.objects.filter(application=application)
|
||||||
|
.values_list('knowledge_id', flat=True)]
|
||||||
|
selected, flow_ids = application_selection(application.knowledge_setting,
|
||||||
|
application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW else None)
|
||||||
|
validate_selection(selected, workspace_id, root_ids + flow_ids)
|
||||||
|
work_flow_version.knowledge_setting = {**(work_flow_version.knowledge_setting or {}), '_knowledge_ids': root_ids}
|
||||||
work_flow_version.save()
|
work_flow_version.save()
|
||||||
access_token = hashlib.md5(
|
access_token = hashlib.md5(
|
||||||
str(uuid.uuid7()).encode()).hexdigest()[
|
str(uuid.uuid7()).encode()).hexdigest()[
|
||||||
|
|
@ -838,6 +840,13 @@ class ApplicationOperateSerializer(serializers.Serializer):
|
||||||
application_id = self.data.get("application_id")
|
application_id = self.data.get("application_id")
|
||||||
|
|
||||||
application = QuerySet(Application).get(id=application_id)
|
application = QuerySet(Application).get(id=application_id)
|
||||||
|
from knowledge.versions import validate_selection, application_selection
|
||||||
|
selected_ids = instance.get('knowledge_id_list')
|
||||||
|
if selected_ids is None:
|
||||||
|
selected_ids = list(ApplicationKnowledgeMapping.objects.filter(application_id=application_id).values_list('knowledge_id', flat=True))
|
||||||
|
selected, flow_ids = application_selection(instance.get('knowledge_setting', application.knowledge_setting),
|
||||||
|
instance.get('work_flow', application.work_flow) if application.type == ApplicationTypeChoices.WORK_FLOW else None)
|
||||||
|
validate_selection(selected, application.workspace_id, list(selected_ids) + flow_ids)
|
||||||
if instance.get('model_id') is None or len(instance.get('model_id')) == 0:
|
if instance.get('model_id') is None or len(instance.get('model_id')) == 0:
|
||||||
application.model_id = None
|
application.model_id = None
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -311,6 +311,9 @@ class ApplicationChatRecordAddKnowledgeSerializer(serializers.Serializer):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
ApplicationChatRecordAddKnowledgeSerializer(data=instance).is_valid(raise_exception=True)
|
ApplicationChatRecordAddKnowledgeSerializer(data=instance).is_valid(raise_exception=True)
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
|
# This endpoint receives its knowledge ID in the body rather than the route.
|
||||||
|
from knowledge.versions import root_knowledge
|
||||||
|
root_knowledge(self.data.get('workspace_id') or 'default', self.data['knowledge_id'])
|
||||||
if scope == 'WORKSPACE':
|
if scope == 'WORKSPACE':
|
||||||
is_permission = get_is_permissions(request=request, workspace_id=self.data.get('workspace_id'),
|
is_permission = get_is_permissions(request=request, workspace_id=self.data.get('workspace_id'),
|
||||||
knowledge_id=self.data.get("knowledge_id"))(
|
knowledge_id=self.data.get("knowledge_id"))(
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,7 @@ class ChatInfo:
|
||||||
self.application = None
|
self.application = None
|
||||||
self.chat_user = None
|
self.chat_user = None
|
||||||
self.debug = debug
|
self.debug = debug
|
||||||
|
self.knowledge_version_map = None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_no_references_setting(knowledge_setting, model_setting):
|
def get_no_references_setting(knowledge_setting, model_setting):
|
||||||
|
|
@ -75,14 +76,14 @@ class ChatInfo:
|
||||||
raise ChatException(500, _('The application does not exist'))
|
raise ChatException(500, _('The application does not exist'))
|
||||||
else:
|
else:
|
||||||
application = QuerySet(ApplicationVersion).filter(application_id=self.application_id).order_by(
|
application = QuerySet(ApplicationVersion).filter(application_id=self.application_id).order_by(
|
||||||
'-create_time')[0:1].first()
|
'-create_time', '-id')[0:1].first()
|
||||||
if not application:
|
if not application:
|
||||||
raise ChatException(500, _("The application has not been published. Please use it after publishing."))
|
raise ChatException(500, _("The application has not been published. Please use it after publishing."))
|
||||||
if application.type == ApplicationTypeChoices.SIMPLE.value:
|
if application.type == ApplicationTypeChoices.SIMPLE.value:
|
||||||
# 数据集id列表
|
# 数据集id列表
|
||||||
knowledge_id_list = [str(row.knowledge_id) for row in
|
published_ids = (application.knowledge_setting or {}).get('_knowledge_ids') if not self.debug else None
|
||||||
QuerySet(ApplicationKnowledgeMapping).filter(
|
knowledge_id_list = published_ids if published_ids is not None else [str(row.knowledge_id) for row in
|
||||||
application_id=self.application_id)]
|
QuerySet(ApplicationKnowledgeMapping).filter(application_id=self.application_id)]
|
||||||
|
|
||||||
# 需要排除的文档
|
# 需要排除的文档
|
||||||
exclude_document_id_list = [str(document.id) for document in
|
exclude_document_id_list = [str(document.id) for document in
|
||||||
|
|
@ -92,6 +93,16 @@ class ChatInfo:
|
||||||
self.knowledge_id_list = knowledge_id_list
|
self.knowledge_id_list = knowledge_id_list
|
||||||
self.exclude_document_id_list = exclude_document_id_list
|
self.exclude_document_id_list = exclude_document_id_list
|
||||||
self.application = application
|
self.application = application
|
||||||
|
from knowledge.versions import pin_selection, validate_selection, application_selection
|
||||||
|
selection, flow_ids = application_selection(application.knowledge_setting,
|
||||||
|
application.work_flow if application.type == ApplicationTypeChoices.WORK_FLOW.value else None)
|
||||||
|
version_knowledge_ids = list(self.knowledge_id_list)
|
||||||
|
version_knowledge_ids.extend(flow_ids)
|
||||||
|
if self.knowledge_version_map is None:
|
||||||
|
validate_selection(selection, application.workspace_id, version_knowledge_ids)
|
||||||
|
self.knowledge_version_map = pin_selection(selection, application.workspace_id, self.chat_id)
|
||||||
|
elif self.knowledge_version_map:
|
||||||
|
pin_selection({}, application.workspace_id, self.chat_id, self.knowledge_version_map)
|
||||||
return application
|
return application
|
||||||
|
|
||||||
def get_chat_user(self, asker=None):
|
def get_chat_user(self, asker=None):
|
||||||
|
|
@ -155,6 +166,7 @@ class ChatInfo:
|
||||||
model_params_setting = credential.get_model_params_setting_form(model.model_name).get_default_form_data()
|
model_params_setting = credential.get_model_params_setting_form(model.model_name).get_default_form_data()
|
||||||
return {
|
return {
|
||||||
'knowledge_id_list': self.knowledge_id_list,
|
'knowledge_id_list': self.knowledge_id_list,
|
||||||
|
'knowledge_version_map': self.knowledge_version_map or {},
|
||||||
'exclude_document_id_list': self.exclude_document_id_list,
|
'exclude_document_id_list': self.exclude_document_id_list,
|
||||||
'exclude_paragraph_id_list': [],
|
'exclude_paragraph_id_list': [],
|
||||||
'top_n': 3 if knowledge_setting.get('top_n') is None else knowledge_setting.get('top_n'),
|
'top_n': 3 if knowledge_setting.get('top_n') is None else knowledge_setting.get('top_n'),
|
||||||
|
|
@ -204,6 +216,7 @@ class ChatInfo:
|
||||||
if not self.debug:
|
if not self.debug:
|
||||||
if not QuerySet(Chat).filter(id=self.chat_id).exists():
|
if not QuerySet(Chat).filter(id=self.chat_id).exists():
|
||||||
Chat(id=self.chat_id, application_id=self.application_id, abstract=question[0:1024],
|
Chat(id=self.chat_id, application_id=self.application_id, abstract=question[0:1024],
|
||||||
|
meta={'knowledge_versions': self.knowledge_version_map or {}},
|
||||||
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
|
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
|
||||||
asker=self.get_chat_user()).save()
|
asker=self.get_chat_user()).save()
|
||||||
|
|
||||||
|
|
@ -244,6 +257,7 @@ class ChatInfo:
|
||||||
if not self.debug:
|
if not self.debug:
|
||||||
if not QuerySet(Chat).filter(id=self.chat_id).exists():
|
if not QuerySet(Chat).filter(id=self.chat_id).exists():
|
||||||
Chat(id=self.chat_id, application_id=self.application_id, abstract=chat_record.problem_text[0:1024],
|
Chat(id=self.chat_id, application_id=self.application_id, abstract=chat_record.problem_text[0:1024],
|
||||||
|
meta={'knowledge_versions': self.knowledge_version_map or {}},
|
||||||
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
|
chat_user_id=self.chat_user_id, chat_user_type=self.chat_user_type,
|
||||||
asker=self.get_chat_user()).save()
|
asker=self.get_chat_user()).save()
|
||||||
else:
|
else:
|
||||||
|
|
@ -260,6 +274,7 @@ class ChatInfo:
|
||||||
'chat_user_type': self.chat_user_type,
|
'chat_user_type': self.chat_user_type,
|
||||||
'knowledge_id_list': self.knowledge_id_list,
|
'knowledge_id_list': self.knowledge_id_list,
|
||||||
'exclude_document_id_list': self.exclude_document_id_list,
|
'exclude_document_id_list': self.exclude_document_id_list,
|
||||||
|
'knowledge_version_map': self.knowledge_version_map,
|
||||||
'application_id': self.application_id,
|
'application_id': self.application_id,
|
||||||
'chat_record_list': [self.chat_record_to_map(c) for c in self.chat_record_list][-20:],
|
'chat_record_list': [self.chat_record_to_map(c) for c in self.chat_record_list][-20:],
|
||||||
'debug': self.debug
|
'debug': self.debug
|
||||||
|
|
@ -309,6 +324,7 @@ class ChatInfo:
|
||||||
chat_info_dict.get('application_id'),
|
chat_info_dict.get('application_id'),
|
||||||
debug=chat_info_dict.get('debug'))
|
debug=chat_info_dict.get('debug'))
|
||||||
c.chat_record_list = [ChatInfo.map_to_chat_record(c_r) for c_r in chat_info_dict.get('chat_record_list')]
|
c.chat_record_list = [ChatInfo.map_to_chat_record(c_r) for c_r in chat_info_dict.get('chat_record_list')]
|
||||||
|
c.knowledge_version_map = chat_info_dict.get('knowledge_version_map')
|
||||||
return c
|
return c
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,6 @@ from (select application."id"::text, application."name",
|
||||||
from application
|
from application
|
||||||
left join "user" on user_id = "user".id
|
left join "user" on user_id = "user".id
|
||||||
where application."id"::text in (select target
|
where application."id"::text in (select target
|
||||||
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
|
from effective_resource_permission ${workspace_user_resource_permission_query_set}
|
||||||
and 'VIEW' = any (permission_list))) temp
|
and 'VIEW' = any (permission_list))) temp
|
||||||
${application_query_set}
|
${application_query_set}
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ from (select application."id"::text, application."name",
|
||||||
from application
|
from application
|
||||||
left join "user" on user_id = "user".id
|
left join "user" on user_id = "user".id
|
||||||
where "application".id::text in (select target
|
where "application".id::text in (select target
|
||||||
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
|
from effective_resource_permission ${workspace_user_resource_permission_query_set}
|
||||||
and case
|
and case
|
||||||
when auth_type = 'ROLE' then
|
when auth_type = 'ROLE' then
|
||||||
'ROLE' = any (permission_list)
|
'ROLE' = any (permission_list)
|
||||||
|
|
@ -28,9 +28,9 @@ from (select application."id"::text, application."name",
|
||||||
from role_permission role_permission
|
from role_permission role_permission
|
||||||
right join user_role_relation user_role_relation
|
right join user_role_relation user_role_relation
|
||||||
on user_role_relation.role_id = role_permission.role_id
|
on user_role_relation.role_id = role_permission.role_id
|
||||||
where user_role_relation.user_id = workspace_user_resource_permission.user_id
|
where user_role_relation.user_id = effective_resource_permission.user_id
|
||||||
and user_role_relation.workspace_id =
|
and user_role_relation.workspace_id =
|
||||||
workspace_user_resource_permission.workspace_id)
|
effective_resource_permission.workspace_id)
|
||||||
|
|
||||||
else
|
else
|
||||||
'VIEW' = any (permission_list)
|
'VIEW' = any (permission_list)
|
||||||
|
|
|
||||||
|
|
@ -272,6 +272,8 @@ class McpServers(APIView):
|
||||||
CompareConstants.AND),
|
CompareConstants.AND),
|
||||||
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
|
||||||
def post(self, request: Request, workspace_id, application_id: str):
|
def post(self, request: Request, workspace_id, application_id: str):
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
enforce(request, 'application', application_id)
|
||||||
return result.success(ApplicationOperateSerializer(
|
return result.success(ApplicationOperateSerializer(
|
||||||
data={'mcp_servers': request.query_params.get('mcp_servers'), 'workspace_id': workspace_id,
|
data={'mcp_servers': request.query_params.get('mcp_servers'), 'workspace_id': workspace_id,
|
||||||
'user_id': request.user.id,
|
'user_id': request.user.id,
|
||||||
|
|
|
||||||
|
|
@ -132,9 +132,11 @@ class OpenView(APIView):
|
||||||
CompareConstants.AND),
|
CompareConstants.AND),
|
||||||
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role())
|
||||||
def get(self, request: Request, workspace_id: str, application_id: str):
|
def get(self, request: Request, workspace_id: str, application_id: str):
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
enforce(request, 'application', application_id)
|
||||||
return result.success(OpenChatSerializers(
|
return result.success(OpenChatSerializers(
|
||||||
data={'workspace_id': workspace_id, 'application_id': application_id,
|
data={'workspace_id': workspace_id, 'application_id': application_id,
|
||||||
'chat_user_id': str(uuid.uuid7()), 'chat_user_type': ChatUserType.ANONYMOUS_USER,
|
'chat_user_id': str(request.user.id), 'chat_user_type': ChatUserType.PLATFORM_USER.value,
|
||||||
'debug': True}).open())
|
'debug': True}).open())
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -152,7 +154,7 @@ class ChatView(APIView):
|
||||||
tags=[_('Application')] # type: ignore
|
tags=[_('Application')] # type: ignore
|
||||||
)
|
)
|
||||||
def post(self, request: Request, chat_id: str):
|
def post(self, request: Request, chat_id: str):
|
||||||
return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data)
|
return DebugChatSerializers(data={'chat_id': chat_id}).chat(request.data, request=request)
|
||||||
|
|
||||||
class PromptGenerateView(APIView):
|
class PromptGenerateView(APIView):
|
||||||
authentication_classes = [TokenAuth]
|
authentication_classes = [TokenAuth]
|
||||||
|
|
|
||||||
|
|
@ -131,10 +131,12 @@ def get_post_handler(chat_info: ChatInfo):
|
||||||
class DebugChatSerializers(serializers.Serializer):
|
class DebugChatSerializers(serializers.Serializer):
|
||||||
chat_id = serializers.UUIDField(required=True, label=_("Conversation ID"))
|
chat_id = serializers.UUIDField(required=True, label=_("Conversation ID"))
|
||||||
|
|
||||||
def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse()):
|
def chat(self, instance: dict, base_to_response: BaseToResponse = SystemToResponse(), request=None):
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
chat_id = self.data.get('chat_id')
|
chat_id = self.data.get('chat_id')
|
||||||
chat_info: ChatInfo = ChatInfo.get_cache(chat_id)
|
chat_info: ChatInfo = ChatInfo.get_cache(chat_id)
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
enforce(request, 'application', chat_info.application_id)
|
||||||
application = QuerySet(Application).filter(id=chat_info.application_id).first()
|
application = QuerySet(Application).filter(id=chat_info.application_id).first()
|
||||||
chat_info.application = application
|
chat_info.application = application
|
||||||
return ChatSerializers(data={
|
return ChatSerializers(data={
|
||||||
|
|
@ -404,7 +406,8 @@ class ChatSerializers(serializers.Serializer):
|
||||||
'workspace_id': workspace_id,
|
'workspace_id': workspace_id,
|
||||||
'debug': debug,
|
'debug': debug,
|
||||||
'chat_user': chat_info.get_chat_user(),
|
'chat_user': chat_info.get_chat_user(),
|
||||||
'application_id': str(chat_info.application_id)},
|
'application_id': str(chat_info.application_id),
|
||||||
|
'knowledge_version_map': chat_info.knowledge_version_map or {}},
|
||||||
WorkFlowPostHandler(chat_info),
|
WorkFlowPostHandler(chat_info),
|
||||||
base_to_response, form_data, image_list, document_list, audio_list,
|
base_to_response, form_data, image_list, document_list, audio_list,
|
||||||
video_list,
|
video_list,
|
||||||
|
|
@ -481,6 +484,9 @@ class ChatSerializers(serializers.Serializer):
|
||||||
is_active=False)]
|
is_active=False)]
|
||||||
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), knowledge_id_list,
|
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), knowledge_id_list,
|
||||||
exclude_document_id_list, application.id)
|
exclude_document_id_list, application.id)
|
||||||
|
previous = Chat.objects.filter(pk=chat_id).first()
|
||||||
|
if previous and 'knowledge_versions' in (previous.meta or {}):
|
||||||
|
chat_info.knowledge_version_map = previous.meta['knowledge_versions']
|
||||||
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
|
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
|
||||||
chat_record_list.sort(key=lambda r: r.create_time)
|
chat_record_list.sort(key=lambda r: r.create_time)
|
||||||
for chat_record in chat_record_list:
|
for chat_record in chat_record_list:
|
||||||
|
|
@ -490,6 +496,9 @@ class ChatSerializers(serializers.Serializer):
|
||||||
def re_open_chat_work_flow(self, chat_id, application):
|
def re_open_chat_work_flow(self, chat_id, application):
|
||||||
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), [], [],
|
chat_info = ChatInfo(chat_id, self.data.get('chat_user_id'), self.data.get('chat_user_type'), [], [],
|
||||||
application.id)
|
application.id)
|
||||||
|
previous = Chat.objects.filter(pk=chat_id).first()
|
||||||
|
if previous and 'knowledge_versions' in (previous.meta or {}):
|
||||||
|
chat_info.knowledge_version_map = previous.meta['knowledge_versions']
|
||||||
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
|
chat_record_list = list(QuerySet(ChatRecord).filter(chat_id=chat_id).order_by('-create_time')[0:5])
|
||||||
chat_record_list.sort(key=lambda r: r.create_time)
|
chat_record_list.sort(key=lambda r: r.create_time)
|
||||||
for chat_record in chat_record_list:
|
for chat_record in chat_record_list:
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,8 @@ class AnonymousAuthenticationSerializer(serializers.Serializer):
|
||||||
access_token = self.data.get("access_token")
|
access_token = self.data.get("access_token")
|
||||||
application_access_token = QuerySet(ApplicationAccessToken).filter(access_token=access_token).first()
|
application_access_token = QuerySet(ApplicationAccessToken).filter(access_token=access_token).first()
|
||||||
if application_access_token is not None and application_access_token.is_active:
|
if application_access_token is not None and application_access_token.is_active:
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
enforce(request, 'application', application_access_token.application_id)
|
||||||
api_key = self.data.get('api_key') or self.get_request_api_key(request)
|
api_key = self.data.get('api_key') or self.get_request_api_key(request)
|
||||||
if api_key:
|
if api_key:
|
||||||
application_api_key = self.validate_application_api_key(application_access_token.application_id, api_key)
|
application_api_key = self.validate_application_api_key(application_access_token.application_id, api_key)
|
||||||
|
|
@ -121,6 +123,8 @@ class AuthProfileSerializer(serializers.Serializer):
|
||||||
if not application_access_token.is_active:
|
if not application_access_token.is_active:
|
||||||
raise NotFound404(404, _("Invalid access_token"))
|
raise NotFound404(404, _("Invalid access_token"))
|
||||||
application_id = application_access_token.application_id
|
application_id = application_access_token.application_id
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
enforce(request, 'application', application_id)
|
||||||
api_key = self.data.get("api_key") or AnonymousAuthenticationSerializer.get_request_api_key(request)
|
api_key = self.data.get("api_key") or AnonymousAuthenticationSerializer.get_request_api_key(request)
|
||||||
if api_key:
|
if api_key:
|
||||||
AnonymousAuthenticationSerializer.validate_application_api_key(application_id, api_key)
|
AnonymousAuthenticationSerializer.validate_application_api_key(application_id, api_key)
|
||||||
|
|
|
||||||
|
|
@ -85,7 +85,19 @@ class TokenAuth(TokenAuthentication):
|
||||||
token_details = TokenDetails(token)
|
token_details = TokenDetails(token)
|
||||||
for handle in handles:
|
for handle in handles:
|
||||||
if handle.support(request, token, token_details.get_token_details):
|
if handle.support(request, token, token_details.get_token_details):
|
||||||
return handle.handle(request, token, token_details.get_token_details)
|
authenticated = handle.handle(request, token, token_details.get_token_details)
|
||||||
|
from knowledge.versions import guard_snapshot_request
|
||||||
|
guard_snapshot_request(request)
|
||||||
|
from common.constants.permission_constants import ChatAuth
|
||||||
|
if isinstance(authenticated[1], ChatAuth):
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
match = getattr(request, 'resolver_match', None)
|
||||||
|
routed_id = match.kwargs.get('application_id') if match else None
|
||||||
|
if routed_id and str(routed_id) != str(authenticated[1].application_id):
|
||||||
|
from common.exception.app_exception import AppUnauthorizedFailed
|
||||||
|
raise AppUnauthorizedFailed(403, '无权访问该助手')
|
||||||
|
enforce(request, 'application', authenticated[1].application_id)
|
||||||
|
return authenticated
|
||||||
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect! illegal user'))
|
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect! illegal user'))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
traceback.print_stack()
|
traceback.print_stack()
|
||||||
|
|
|
||||||
|
|
@ -275,6 +275,10 @@ def get_role_list(user,
|
||||||
|
|
||||||
|
|
||||||
def get_auth(user):
|
def get_auth(user):
|
||||||
|
from system_manage.platform_access import is_admin
|
||||||
|
# The old field is a compatibility projection, never an authority.
|
||||||
|
administrator = is_admin(user)
|
||||||
|
user.role = 'ADMIN' if administrator else 'USER'
|
||||||
workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping")
|
workspace_user_role_mapping_model = DatabaseModelManage.get_model("workspace_user_role_mapping")
|
||||||
workspace_model = DatabaseModelManage.get_model("workspace_model")
|
workspace_model = DatabaseModelManage.get_model("workspace_model")
|
||||||
role_model = DatabaseModelManage.get_model("role_model")
|
role_model = DatabaseModelManage.get_model("role_model")
|
||||||
|
|
@ -284,7 +288,15 @@ def get_auth(user):
|
||||||
role_model, role_permission_mapping_model)
|
role_model, role_permission_mapping_model)
|
||||||
role_list = get_role_list(user, workspace_user_role_mapping_model, workspace_model,
|
role_list = get_role_list(user, workspace_user_role_mapping_model, workspace_model,
|
||||||
role_model, role_permission_mapping_model)
|
role_model, role_permission_mapping_model)
|
||||||
return Auth(role_list, permission_list)
|
if workspace_user_role_mapping_model is None:
|
||||||
|
role_list = ['ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')] if administrator else ['USER', get_role_permission(RoleConstants.USER, 'default')]
|
||||||
|
elif administrator:
|
||||||
|
role_list = list(set(role_list) | {'ADMIN', get_role_permission(RoleConstants.WORKSPACE_MANAGE, 'default')})
|
||||||
|
else:
|
||||||
|
role_list = [role for role in role_list if role not in ('ADMIN', 'EXTENDS_ADMIN')]
|
||||||
|
from system_manage.platform_access import permission_strings
|
||||||
|
# Evaluate new grants live, independently of legacy cached permissions.
|
||||||
|
return Auth(role_list, list(set(permission_list) | permission_strings(user)))
|
||||||
|
|
||||||
|
|
||||||
class UserToken(AuthBaseHandle):
|
class UserToken(AuthBaseHandle):
|
||||||
|
|
@ -306,4 +318,6 @@ class UserToken(AuthBaseHandle):
|
||||||
if not user.is_active or user.password != cache_token.password:
|
if not user.is_active or user.password != cache_token.password:
|
||||||
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect'))
|
raise AppAuthenticationFailed(1002, _('Authentication information is incorrect'))
|
||||||
auth = get_auth(user)
|
auth = get_auth(user)
|
||||||
|
from system_manage.platform_access import check_feature_request
|
||||||
|
check_feature_request(request, user)
|
||||||
return user, auth
|
return user, auth
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,9 @@
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
class MKTokenizer:
|
class MKTokenizer:
|
||||||
def __init__(self, tokenizer):
|
def __init__(self, tokenizer):
|
||||||
|
|
@ -17,14 +20,38 @@ class MKTokenizer:
|
||||||
return self.tokenizer.encode(text).ids
|
return self.tokenizer.encode(text).ids
|
||||||
|
|
||||||
|
|
||||||
|
class ByteTokenEstimate:
|
||||||
|
"""Conservative token budget when an offline tokenizer is unavailable.
|
||||||
|
|
||||||
|
Provider-reported usage remains authoritative. This only supplies local
|
||||||
|
estimates for prompt budgeting; it never downloads model files.
|
||||||
|
"""
|
||||||
|
def encode(self, text):
|
||||||
|
return list(text.encode('utf-8'))
|
||||||
|
|
||||||
|
|
||||||
class TokenizerManage:
|
class TokenizerManage:
|
||||||
tokenizer = None
|
tokenizer = None
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_tokenizer():
|
def get_tokenizer():
|
||||||
|
if TokenizerManage.tokenizer is not None:
|
||||||
|
return TokenizerManage.tokenizer
|
||||||
|
with TokenizerManage._lock:
|
||||||
|
if TokenizerManage.tokenizer is not None:
|
||||||
|
return TokenizerManage.tokenizer
|
||||||
from tokenizers import Tokenizer
|
from tokenizers import Tokenizer
|
||||||
# 创建Tokenizer
|
model_path = Path(os.environ.get('MAXKB_TOKENIZER_PATH') or
|
||||||
model_path = os.path.join("/opt/maxkb-app", "model", "tokenizer", "models--bert-base-cased")
|
'/opt/maxkb-app/model/tokenizer/models--bert-base-cased')
|
||||||
with open(f"{model_path}/refs/main", encoding="utf-8") as f: snapshot = f.read()
|
try:
|
||||||
TokenizerManage.tokenizer = Tokenizer.from_file(f"{model_path}/snapshots/{snapshot}/tokenizer.json")
|
path = model_path if model_path.is_file() else model_path / 'tokenizer.json'
|
||||||
return MKTokenizer(TokenizerManage.tokenizer)
|
if not path.is_file():
|
||||||
|
snapshot = (model_path / 'refs/main').read_text(encoding='utf-8').strip()
|
||||||
|
path = model_path / 'snapshots' / snapshot / 'tokenizer.json'
|
||||||
|
TokenizerManage.tokenizer = MKTokenizer(Tokenizer.from_file(str(path)))
|
||||||
|
except Exception:
|
||||||
|
# A missing/corrupt tokenizer must not prevent remote-model chat.
|
||||||
|
logging.getLogger(__name__).warning('本地分词器不可用,使用保守字节估算;实际用量以模型返回为准')
|
||||||
|
TokenizerManage.tokenizer = ByteTokenEstimate()
|
||||||
|
return TokenizerManage.tokenizer
|
||||||
|
|
|
||||||
|
|
@ -357,7 +357,7 @@ class ListenerManagement:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def update_problem(args: UpdateProblemArgs):
|
def update_problem(args: UpdateProblemArgs):
|
||||||
problem_paragraph_mapping_list = QuerySet(ProblemParagraphMapping).filter(problem_id=args.problem_id)
|
problem_paragraph_mapping_list = QuerySet(ProblemParagraphMapping).filter(problem_id=args.problem_id)
|
||||||
embed_value = args.embedding_model.embed_query(args.problem_content)
|
embed_value = args.embedding_model.embed_documents([args.problem_content])[0]
|
||||||
VectorStore.get_embedding_vector().update_by_source_ids([v.id for v in problem_paragraph_mapping_list],
|
VectorStore.get_embedding_vector().update_by_source_ids([v.id for v in problem_paragraph_mapping_list],
|
||||||
{'embedding': embed_value})
|
{'embedding': embed_value})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,6 @@ import logging
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
from rest_framework.exceptions import ValidationError, ErrorDetail, APIException
|
from rest_framework.exceptions import ValidationError, ErrorDetail, APIException
|
||||||
from rest_framework.utils.serializer_helpers import ReturnDict
|
|
||||||
from rest_framework.views import exception_handler
|
from rest_framework.views import exception_handler
|
||||||
|
|
||||||
from common import result
|
from common import result
|
||||||
|
|
@ -60,28 +59,18 @@ def validation_error_to_result(exc: ValidationError):
|
||||||
|
|
||||||
|
|
||||||
def find_err_detail(exc_detail):
|
def find_err_detail(exc_detail):
|
||||||
if isinstance(exc_detail, ErrorDetail):
|
if isinstance(exc_detail, (ErrorDetail, str)):
|
||||||
return exc_detail
|
return exc_detail
|
||||||
if isinstance(exc_detail, dict):
|
if isinstance(exc_detail, dict):
|
||||||
keys = exc_detail.keys()
|
for key, value in exc_detail.items():
|
||||||
for key in keys:
|
detail = find_err_detail(value)
|
||||||
_label = get_label(key, exc_detail)
|
if detail is not None:
|
||||||
_value = exc_detail[key]
|
if key == 'non_field_errors':
|
||||||
if isinstance(_value, list):
|
return detail
|
||||||
for v in _value:
|
return f'{get_label(key, exc_detail)}:{detail}'
|
||||||
r = find_err_detail(ReturnDict({key: v}, serializer=exc_detail.serializer))
|
if isinstance(exc_detail, (list, tuple)):
|
||||||
if r is not None:
|
|
||||||
return r
|
|
||||||
if isinstance(_value, ErrorDetail):
|
|
||||||
return f"{_label}:{find_err_detail(_value)}"
|
|
||||||
if isinstance(_value, dict) and len(_value.keys()) > 0:
|
|
||||||
try:
|
|
||||||
return find_err_detail(ReturnDict(_value, serializer=exc_detail.serializer.fields[key]))
|
|
||||||
except Exception as e:
|
|
||||||
return _value
|
|
||||||
if isinstance(exc_detail, list):
|
|
||||||
for v in exc_detail:
|
for v in exc_detail:
|
||||||
r = find_err_detail(ReturnDict(v, serializer=exc_detail.serializer.child))
|
r = find_err_detail(v)
|
||||||
if r is not None:
|
if r is not None:
|
||||||
return r
|
return r
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
import json
|
||||||
|
from django.test import SimpleTestCase
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
|
from common.exception.handle_exception import validation_error_to_result
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationMessageTests(SimpleTestCase):
|
||||||
|
def test_plain_list_error_is_readable(self):
|
||||||
|
response = validation_error_to_result(ValidationError('最多可保存 10 张导图,请先删除不需要的导图'))
|
||||||
|
self.assertEqual(json.loads(response.content)['message'], '最多可保存 10 张导图,请先删除不需要的导图')
|
||||||
|
|
||||||
|
def test_nested_error_retains_field_context(self):
|
||||||
|
response = validation_error_to_result(ValidationError({'document': {'name': ['不能为空']}}))
|
||||||
|
self.assertEqual(json.loads(response.content)['message'], 'document:name:不能为空')
|
||||||
|
|
||||||
|
def test_non_field_error_has_no_internal_key(self):
|
||||||
|
response = validation_error_to_result(ValidationError({'non_field_errors': ['内容无效']}))
|
||||||
|
self.assertEqual(json.loads(response.content)['message'], '内容无效')
|
||||||
|
|
@ -22,3 +22,4 @@ from .table_checkbox import *
|
||||||
from .radio_card_field import *
|
from .radio_card_field import *
|
||||||
from .label import *
|
from .label import *
|
||||||
from .slider_field import *
|
from .slider_field import *
|
||||||
|
from .switch_field import *
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ from charset_normalizer import detect
|
||||||
|
|
||||||
from common.handle.base_split_handle import BaseSplitHandle
|
from common.handle.base_split_handle import BaseSplitHandle
|
||||||
from common.utils.logger import maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
from common.utils.table_split import split_table_rows
|
||||||
|
|
||||||
|
|
||||||
def post_cell(cell_value):
|
def post_cell(cell_value):
|
||||||
|
|
@ -28,7 +29,7 @@ def row_to_md(row):
|
||||||
|
|
||||||
|
|
||||||
class CsvSplitHandle(BaseSplitHandle):
|
class CsvSplitHandle(BaseSplitHandle):
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
buffer = get_buffer(file)
|
buffer = get_buffer(file)
|
||||||
paragraphs = []
|
paragraphs = []
|
||||||
file_name = os.path.basename(file.name)
|
file_name = os.path.basename(file.name)
|
||||||
|
|
@ -46,22 +47,7 @@ class CsvSplitHandle(BaseSplitHandle):
|
||||||
return result
|
return result
|
||||||
if len(title_row_list) == 0:
|
if len(title_row_list) == 0:
|
||||||
return result
|
return result
|
||||||
result_item_content = ''
|
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in reader), limit, overlap))
|
||||||
for row in reader:
|
|
||||||
next_md_content = row_to_md(row)
|
|
||||||
next_md_content_len = len(next_md_content)
|
|
||||||
result_item_content_len = len(result_item_content)
|
|
||||||
if len(result_item_content) == 0:
|
|
||||||
result_item_content += title_md_content
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
if result_item_content_len + next_md_content_len < limit:
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
result_item_content = title_md_content + next_md_content
|
|
||||||
if len(result_item_content) > 0:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
return result
|
return result
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_logger.error(f"Error processing CSV file {file.name}: {e}, {traceback.format_exc()}")
|
maxkb_logger.error(f"Error processing CSV file {file.name}: {e}, {traceback.format_exc()}")
|
||||||
|
|
|
||||||
|
|
@ -192,7 +192,7 @@ class DocSplitHandle(BaseSplitHandle):
|
||||||
for element
|
for element
|
||||||
in elements])
|
in elements])
|
||||||
|
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
file_name = os.path.basename(file.name)
|
file_name = os.path.basename(file.name)
|
||||||
try:
|
try:
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
|
|
@ -206,9 +206,9 @@ class DocSplitHandle(BaseSplitHandle):
|
||||||
if len(image_list) > 0:
|
if len(image_list) > 0:
|
||||||
save_image(image_list)
|
save_image(image_list)
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
|
||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")
|
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -44,16 +44,16 @@ class HTMLSplitHandle(BaseSplitHandle):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
buffer = get_buffer(file)
|
buffer = get_buffer(file)
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
if type(with_filter) is str:
|
if type(with_filter) is str:
|
||||||
with_filter = with_filter.lower() == 'true'
|
with_filter = with_filter.lower() == 'true'
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
|
||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
|
||||||
try:
|
try:
|
||||||
encoding = get_encoding(buffer)
|
encoding = get_encoding(buffer)
|
||||||
content = buffer.decode(encoding)
|
content = buffer.decode(encoding)
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ def check_links_in_pdf(doc):
|
||||||
|
|
||||||
|
|
||||||
class PdfSplitHandle(BaseSplitHandle):
|
class PdfSplitHandle(BaseSplitHandle):
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
|
||||||
# 将上传的文件保存到临时文件中
|
# 将上传的文件保存到临时文件中
|
||||||
for chunk in file.chunks():
|
for chunk in file.chunks():
|
||||||
|
|
@ -57,12 +57,12 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
if type(with_filter) is str:
|
if type(with_filter) is str:
|
||||||
with_filter = with_filter.lower() == 'true'
|
with_filter = with_filter.lower() == 'true'
|
||||||
# 处理有目录的pdf
|
# 处理有目录的pdf
|
||||||
result = self.handle_toc(pdf_document, limit)
|
result = self.handle_toc(pdf_document, limit, overlap)
|
||||||
if result is not None:
|
if result is not None:
|
||||||
return {'name': file.name, 'content': result}
|
return {'name': file.name, 'content': result}
|
||||||
|
|
||||||
# 没目录但是有链接的pdf
|
# 没目录但是有链接的pdf
|
||||||
result = self.handle_links(pdf_document, pattern_list, with_filter, limit)
|
result = self.handle_links(pdf_document, pattern_list, with_filter, limit, overlap)
|
||||||
if result is not None and len(result) > 0:
|
if result is not None and len(result) > 0:
|
||||||
return {'name': file.name, 'content': result}
|
return {'name': file.name, 'content': result}
|
||||||
|
|
||||||
|
|
@ -70,9 +70,9 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
content = self.handle_pdf_content(file, pdf_document)
|
content = self.handle_pdf_content(file, pdf_document)
|
||||||
|
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
|
||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_logger.error(f"File: {file.name}, error: {e}, {traceback.format_exc()}")
|
maxkb_logger.error(f"File: {file.name}, error: {e}, {traceback.format_exc()}")
|
||||||
return {
|
return {
|
||||||
|
|
@ -131,7 +131,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
return content
|
return content
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def handle_toc(doc, limit):
|
def handle_toc(doc, limit, overlap: int = 0):
|
||||||
# 找到目录
|
# 找到目录
|
||||||
toc = doc.get_toc()
|
toc = doc.get_toc()
|
||||||
if toc is None or len(toc) == 0:
|
if toc is None or len(toc) == 0:
|
||||||
|
|
@ -183,7 +183,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
real_chapter_title = chapter_title[:256]
|
real_chapter_title = chapter_title[:256]
|
||||||
# 限制章节内容长度
|
# 限制章节内容长度
|
||||||
if 0 < limit < len(chapter_text):
|
if 0 < limit < len(chapter_text):
|
||||||
split_text = smart_split_paragraph(chapter_text, limit)
|
split_text = smart_split_paragraph(chapter_text, limit, overlap)
|
||||||
for text in split_text:
|
for text in split_text:
|
||||||
chapters.append({"title": real_chapter_title, "content": text})
|
chapters.append({"title": real_chapter_title, "content": text})
|
||||||
else:
|
else:
|
||||||
|
|
@ -193,7 +193,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
return chapters
|
return chapters
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def handle_links(doc, pattern_list, with_filter, limit):
|
def handle_links(doc, pattern_list, with_filter, limit, overlap: int = 0):
|
||||||
# 检查文档是否包含内部链接
|
# 检查文档是否包含内部链接
|
||||||
if not check_links_in_pdf(doc):
|
if not check_links_in_pdf(doc):
|
||||||
return
|
return
|
||||||
|
|
@ -262,7 +262,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
|
|
||||||
# 限制章节内容长度
|
# 限制章节内容长度
|
||||||
if 0 < limit < len(chapter_text):
|
if 0 < limit < len(chapter_text):
|
||||||
split_text = smart_split_paragraph(chapter_text, limit)
|
split_text = smart_split_paragraph(chapter_text, limit, overlap)
|
||||||
for text in split_text:
|
for text in split_text:
|
||||||
chapters.append({"title": link_title, "content": text})
|
chapters.append({"title": link_title, "content": text})
|
||||||
else:
|
else:
|
||||||
|
|
@ -285,9 +285,9 @@ class PdfSplitHandle(BaseSplitHandle):
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
|
maxkb_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
|
||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
|
||||||
# 插入目录前的部分
|
# 插入目录前的部分
|
||||||
page_content = re.sub(r'(?<!。)\n+', '', page_content)
|
page_content = re.sub(r'(?<!。)\n+', '', page_content)
|
||||||
page_content = re.sub(r'(?<!.)\n+', '', page_content)
|
page_content = re.sub(r'(?<!.)\n+', '', page_content)
|
||||||
|
|
|
||||||
|
|
@ -39,16 +39,16 @@ class TextSplitHandle(BaseSplitHandle):
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
buffer = get_buffer(file)
|
buffer = get_buffer(file)
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
if type(with_filter) is str:
|
if type(with_filter) is str:
|
||||||
with_filter = with_filter.lower() == 'true'
|
with_filter = with_filter.lower() == 'true'
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit, overlap)
|
||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap)
|
||||||
try:
|
try:
|
||||||
content = buffer.decode(detect(buffer)['encoding'])
|
content = buffer.decode(detect(buffer)['encoding'])
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ import xlrd
|
||||||
|
|
||||||
from common.handle.base_split_handle import BaseSplitHandle
|
from common.handle.base_split_handle import BaseSplitHandle
|
||||||
from common.utils.logger import maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
from common.utils.table_split import split_table_rows
|
||||||
|
|
||||||
|
|
||||||
def post_cell(cell_value):
|
def post_cell(cell_value):
|
||||||
|
|
@ -24,7 +25,7 @@ def row_to_md(row):
|
||||||
[post_cell(str(cell)) if cell is not None else '' for cell in row]) + ' |\n'
|
[post_cell(str(cell)) if cell is not None else '' for cell in row]) + ' |\n'
|
||||||
|
|
||||||
|
|
||||||
def handle_sheet(file_name, sheet, limit: int):
|
def handle_sheet(file_name, sheet, limit: int, overlap: int = 0):
|
||||||
rows = iter([sheet.row_values(i) for i in range(sheet.nrows)])
|
rows = iter([sheet.row_values(i) for i in range(sheet.nrows)])
|
||||||
paragraphs = []
|
paragraphs = []
|
||||||
result = {'name': file_name, 'content': paragraphs}
|
result = {'name': file_name, 'content': paragraphs}
|
||||||
|
|
@ -37,27 +38,12 @@ def handle_sheet(file_name, sheet, limit: int):
|
||||||
return result
|
return result
|
||||||
if len(title_row_list) == 0:
|
if len(title_row_list) == 0:
|
||||||
return result
|
return result
|
||||||
result_item_content = ''
|
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in rows), limit, overlap))
|
||||||
for row in rows:
|
|
||||||
next_md_content = row_to_md(row)
|
|
||||||
next_md_content_len = len(next_md_content)
|
|
||||||
result_item_content_len = len(result_item_content)
|
|
||||||
if len(result_item_content) == 0:
|
|
||||||
result_item_content += title_md_content
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
if result_item_content_len + next_md_content_len < limit:
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
result_item_content = title_md_content + next_md_content
|
|
||||||
if len(result_item_content) > 0:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
class XlsSplitHandle(BaseSplitHandle):
|
class XlsSplitHandle(BaseSplitHandle):
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
buffer = get_buffer(file)
|
buffer = get_buffer(file)
|
||||||
try:
|
try:
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
|
|
@ -67,8 +53,8 @@ class XlsSplitHandle(BaseSplitHandle):
|
||||||
worksheets_size = len(worksheets)
|
worksheets_size = len(worksheets)
|
||||||
return [row for row in
|
return [row for row in
|
||||||
[handle_sheet(file.name,
|
[handle_sheet(file.name,
|
||||||
sheet, limit) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet(
|
sheet, limit, overlap) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet(
|
||||||
sheet.name, sheet, limit) for sheet
|
sheet.name, sheet, limit, overlap) for sheet
|
||||||
in worksheets] if row is not None]
|
in worksheets] if row is not None]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_logger.error(f"Error processing XLS file {file.name}: {e}, {traceback.format_exc()}")
|
maxkb_logger.error(f"Error processing XLS file {file.name}: {e}, {traceback.format_exc()}")
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ import openpyxl
|
||||||
from common.handle.base_split_handle import BaseSplitHandle
|
from common.handle.base_split_handle import BaseSplitHandle
|
||||||
from common.handle.impl.common_handle import xlsx_embed_cells_images
|
from common.handle.impl.common_handle import xlsx_embed_cells_images
|
||||||
from common.utils.logger import maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
from common.utils.table_split import split_table_rows
|
||||||
|
|
||||||
|
|
||||||
def post_cell(image_dict, cell_value):
|
def post_cell(image_dict, cell_value):
|
||||||
|
|
@ -30,7 +31,7 @@ def row_to_md(row, image_dict):
|
||||||
in row]) + ' |\n'
|
in row]) + ' |\n'
|
||||||
|
|
||||||
|
|
||||||
def handle_sheet(file_name, sheet, image_dict, limit: int):
|
def handle_sheet(file_name, sheet, image_dict, limit: int, overlap: int = 0):
|
||||||
rows = sheet.rows
|
rows = sheet.rows
|
||||||
paragraphs = []
|
paragraphs = []
|
||||||
result = {'name': file_name, 'content': paragraphs}
|
result = {'name': file_name, 'content': paragraphs}
|
||||||
|
|
@ -43,32 +44,17 @@ def handle_sheet(file_name, sheet, image_dict, limit: int):
|
||||||
return result
|
return result
|
||||||
if len(title_row_list) == 0:
|
if len(title_row_list) == 0:
|
||||||
return result
|
return result
|
||||||
result_item_content = ''
|
paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row, image_dict) for row in rows), limit, overlap))
|
||||||
for row in rows:
|
|
||||||
next_md_content = row_to_md(row, image_dict)
|
|
||||||
next_md_content_len = len(next_md_content)
|
|
||||||
result_item_content_len = len(result_item_content)
|
|
||||||
if len(result_item_content) == 0:
|
|
||||||
result_item_content += title_md_content
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
if result_item_content_len + next_md_content_len < limit:
|
|
||||||
result_item_content += next_md_content
|
|
||||||
else:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
result_item_content = title_md_content + next_md_content
|
|
||||||
if len(result_item_content) > 0:
|
|
||||||
paragraphs.append({'content': result_item_content, 'title': ''})
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
class XlsxSplitHandle(BaseSplitHandle):
|
class XlsxSplitHandle(BaseSplitHandle):
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
buffer = get_buffer(file)
|
buffer = get_buffer(file)
|
||||||
try:
|
try:
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
workbook = openpyxl.load_workbook(io.BytesIO(buffer))
|
workbook = openpyxl.load_workbook(io.BytesIO(buffer), data_only=True)
|
||||||
try:
|
try:
|
||||||
image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer))
|
image_dict: dict = xlsx_embed_cells_images(io.BytesIO(buffer))
|
||||||
save_image([item for item in image_dict.values()])
|
save_image([item for item in image_dict.values()])
|
||||||
|
|
@ -80,8 +66,8 @@ class XlsxSplitHandle(BaseSplitHandle):
|
||||||
[handle_sheet(file.name,
|
[handle_sheet(file.name,
|
||||||
sheet,
|
sheet,
|
||||||
image_dict,
|
image_dict,
|
||||||
limit) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet(
|
limit, overlap) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet(
|
||||||
sheet.title, sheet, image_dict, limit) for sheet
|
sheet.title, sheet, image_dict, limit, overlap) for sheet
|
||||||
in worksheets] if row is not None]
|
in worksheets] if row is not None]
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")
|
maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}")
|
||||||
|
|
|
||||||
|
|
@ -50,11 +50,11 @@ split_handles = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image):
|
def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image, overlap: int = 0):
|
||||||
get_buffer = FileBufferHandle().get_buffer
|
get_buffer = FileBufferHandle().get_buffer
|
||||||
for split_handle in split_handles:
|
for split_handle in split_handles:
|
||||||
if split_handle.support(file, get_buffer):
|
if split_handle.support(file, get_buffer):
|
||||||
return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image)
|
return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image, overlap)
|
||||||
raise Exception(_('Unsupported file format'))
|
raise Exception(_('Unsupported file format'))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -116,7 +116,7 @@ def filter_image_file(result_list: list, image_list):
|
||||||
|
|
||||||
|
|
||||||
class ZipSplitHandle(BaseSplitHandle):
|
class ZipSplitHandle(BaseSplitHandle):
|
||||||
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image):
|
def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0):
|
||||||
if type(limit) is str:
|
if type(limit) is str:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
if type(with_filter) is str:
|
if type(with_filter) is str:
|
||||||
|
|
@ -137,7 +137,7 @@ class ZipSplitHandle(BaseSplitHandle):
|
||||||
try:
|
try:
|
||||||
# 处理一下文件名
|
# 处理一下文件名
|
||||||
f.name = get_file_name(f.name)
|
f.name = get_file_name(f.name)
|
||||||
value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image)
|
value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image, overlap)
|
||||||
if isinstance(value, list):
|
if isinstance(value, list):
|
||||||
result = [*result, *value]
|
result = [*result, *value]
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,8 @@ class GunicornLocalModelService(BaseService):
|
||||||
print("\n- Start Gunicorn Local Model WSGI HTTP Server")
|
print("\n- Start Gunicorn Local Model WSGI HTTP Server")
|
||||||
log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
|
log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
|
||||||
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
|
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
|
||||||
worker = CONFIG.get("LOCAL_MODEL_HOST_WORKER", 1)
|
# Weights and the load queue have one owner; inference remains multithreaded.
|
||||||
|
worker = 1
|
||||||
max_requests = 10240 if int(worker) > 1 else 0
|
max_requests = 10240 if int(worker) > 1 else 0
|
||||||
cmd = [
|
cmd = [
|
||||||
'gunicorn', 'maxkb.wsgi:application',
|
'gunicorn', 'maxkb.wsgi:application',
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,202 @@
|
||||||
|
"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
|
||||||
|
import contextlib
|
||||||
|
import itertools
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from concurrent.futures import Future
|
||||||
|
|
||||||
|
|
||||||
|
def loading_policy(value=None):
|
||||||
|
value = {} if value is None else value
|
||||||
|
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
|
||||||
|
raise ValueError('加载策略无效')
|
||||||
|
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
|
||||||
|
'idle_seconds': value.get('idle_seconds', 300)}
|
||||||
|
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
|
||||||
|
raise ValueError('加载方式无效')
|
||||||
|
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
|
||||||
|
raise ValueError('加载优先级须为0~100的整数')
|
||||||
|
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
|
||||||
|
raise ValueError('空闲时间须为30~86400秒')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class ModelRuntime:
|
||||||
|
def __init__(self, specs, adapter, clock=time.monotonic):
|
||||||
|
self.specs, self.adapter, self.clock = specs, adapter, clock
|
||||||
|
self.entries = {}
|
||||||
|
self.lock = threading.RLock()
|
||||||
|
self.queue = queue.PriorityQueue()
|
||||||
|
self.sequence = itertools.count()
|
||||||
|
self.load_sequence = itertools.count(1)
|
||||||
|
self.worker = None
|
||||||
|
|
||||||
|
def _entry(self, model_id):
|
||||||
|
model_id = str(model_id)
|
||||||
|
spec = self.specs(model_id)
|
||||||
|
spec['policy'] = loading_policy(spec.get('policy'))
|
||||||
|
entry = self.entries.get(model_id)
|
||||||
|
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
|
||||||
|
if entry['leases'] or entry['state'] == 'LOADING':
|
||||||
|
raise ValueError('模型配置已变更,请等待当前请求结束')
|
||||||
|
self._discard(entry)
|
||||||
|
entry = None
|
||||||
|
if entry is None:
|
||||||
|
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
|
||||||
|
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
|
||||||
|
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
|
||||||
|
'load_order': None}
|
||||||
|
self.entries[model_id] = entry
|
||||||
|
previous = entry['spec']['policy']
|
||||||
|
entry['spec'] = spec
|
||||||
|
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
|
||||||
|
entry['generation'] += 1
|
||||||
|
self._put(entry)
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def _put(self, entry):
|
||||||
|
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
with self.lock:
|
||||||
|
if self.worker is None:
|
||||||
|
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
|
||||||
|
self.worker.start()
|
||||||
|
|
||||||
|
def submit(self, model_id, start=True):
|
||||||
|
with self.lock:
|
||||||
|
entry = self._entry(model_id)
|
||||||
|
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
|
||||||
|
entry.update(state='UNLOADED', value=None, info={})
|
||||||
|
if entry['state'] == 'READY':
|
||||||
|
return entry
|
||||||
|
if entry['state'] not in ('LOADING', 'QUEUED'):
|
||||||
|
entry.update(state='QUEUED', error='', future=Future())
|
||||||
|
entry['generation'] += 1
|
||||||
|
self._put(entry)
|
||||||
|
if start:
|
||||||
|
self.start()
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def preload(self, model_ids):
|
||||||
|
# Enqueue the complete startup batch before choosing the highest-priority model.
|
||||||
|
with self.lock:
|
||||||
|
for model_id in model_ids:
|
||||||
|
self.submit(model_id, start=False)
|
||||||
|
self.start()
|
||||||
|
|
||||||
|
def _work(self):
|
||||||
|
while True:
|
||||||
|
_, order, model_id, generation = self.queue.get()
|
||||||
|
try:
|
||||||
|
with self.lock:
|
||||||
|
entry = self.entries.get(model_id)
|
||||||
|
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
|
||||||
|
continue
|
||||||
|
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
|
||||||
|
spec = entry['spec']
|
||||||
|
future = entry['future']
|
||||||
|
value, info = self.adapter.load(spec)
|
||||||
|
with self.lock:
|
||||||
|
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
|
||||||
|
future.set_result(True)
|
||||||
|
del value # The loader thread must not keep the last model alive after eviction.
|
||||||
|
except Exception as exc:
|
||||||
|
with self.lock:
|
||||||
|
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
|
||||||
|
if not entry['future'].done():
|
||||||
|
entry['future'].set_exception(ValueError(entry['error']))
|
||||||
|
finally:
|
||||||
|
self.queue.task_done()
|
||||||
|
|
||||||
|
def acquire(self, model_id, remote=False, timeout=600):
|
||||||
|
with self.lock:
|
||||||
|
entry = self.submit(model_id)
|
||||||
|
token = uuid.uuid4().hex
|
||||||
|
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
|
||||||
|
future = entry['future'] if entry['state'] != 'READY' else None
|
||||||
|
try:
|
||||||
|
if future:
|
||||||
|
future.result(timeout=timeout)
|
||||||
|
return token
|
||||||
|
except Exception:
|
||||||
|
self.release(model_id, token)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def release(self, model_id, token):
|
||||||
|
with self.lock:
|
||||||
|
entry = self.entries.get(str(model_id))
|
||||||
|
if entry:
|
||||||
|
entry['leases'].pop(token, None)
|
||||||
|
entry['last_used'] = self.clock()
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def inference(self, model_id):
|
||||||
|
token = self.acquire(model_id)
|
||||||
|
try:
|
||||||
|
with self.lock:
|
||||||
|
value = self.entries[str(model_id)]['value']
|
||||||
|
yield value
|
||||||
|
finally:
|
||||||
|
self.release(model_id, token)
|
||||||
|
|
||||||
|
def _discard(self, entry):
|
||||||
|
if entry['value'] is not None:
|
||||||
|
value = entry['value']
|
||||||
|
self.adapter.unload(entry['spec'], value)
|
||||||
|
entry['value'] = None
|
||||||
|
del value
|
||||||
|
self.adapter.collect()
|
||||||
|
entry.update(state='UNLOADED', loaded_at=None, info={})
|
||||||
|
|
||||||
|
def unload(self, model_id):
|
||||||
|
with self.lock:
|
||||||
|
entry = self._entry(model_id)
|
||||||
|
self._expire_leases(entry)
|
||||||
|
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
|
||||||
|
raise ValueError('模型正在使用或加载,请稍后再试')
|
||||||
|
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
|
||||||
|
entry['value'] = True
|
||||||
|
self._discard(entry)
|
||||||
|
|
||||||
|
def configure(self, model_id):
|
||||||
|
with self.lock:
|
||||||
|
entry = self._entry(model_id)
|
||||||
|
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
|
||||||
|
if warm_again:
|
||||||
|
entry.update(state='UNLOADED', value=None)
|
||||||
|
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
|
||||||
|
self.submit(model_id)
|
||||||
|
|
||||||
|
def _expire_leases(self, entry):
|
||||||
|
now = self.clock()
|
||||||
|
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
|
||||||
|
|
||||||
|
def sweep(self):
|
||||||
|
with self.lock:
|
||||||
|
for entry in list(self.entries.values()):
|
||||||
|
self._expire_leases(entry)
|
||||||
|
policy = entry['spec']['policy']
|
||||||
|
# Ollama owns its idle timer; do not unload models used by other clients.
|
||||||
|
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
|
||||||
|
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
|
||||||
|
self.clock() - entry['last_used'] >= policy['idle_seconds']):
|
||||||
|
self._discard(entry)
|
||||||
|
|
||||||
|
def status(self, model_id):
|
||||||
|
with self.lock:
|
||||||
|
entry = self._entry(model_id)
|
||||||
|
self._expire_leases(entry)
|
||||||
|
# Reconcile native residency, including expiration outside this process.
|
||||||
|
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
|
||||||
|
info = self.adapter.probe(entry['spec'])
|
||||||
|
if info:
|
||||||
|
entry.update(state='READY', info=info, value=True)
|
||||||
|
elif entry['state'] == 'READY':
|
||||||
|
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
|
||||||
|
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
|
||||||
|
'active_requests': len(entry['leases']), 'error': entry['error'],
|
||||||
|
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
|
||||||
|
'load_order': entry['load_order'], **entry['info']}
|
||||||
|
|
@ -0,0 +1,89 @@
|
||||||
|
"""Authenticated internal model-service RPC; never forwards user-supplied endpoints."""
|
||||||
|
import contextlib
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from functools import lru_cache
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def _secret():
|
||||||
|
if os.environ.get('SERVER_NAME') == 'local_model':
|
||||||
|
from local_model.models.system_setting import SystemSetting
|
||||||
|
else:
|
||||||
|
from system_manage.models import SystemSetting
|
||||||
|
meta = SystemSetting.objects.filter(type=1).values_list('meta', flat=True).first()
|
||||||
|
if not meta or not meta.get('value'):
|
||||||
|
raise ValueError('模型服务认证未配置')
|
||||||
|
return hashlib.sha256(('model-runtime-v1:' + meta['value']).encode()).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def signed_request(payload):
|
||||||
|
data = {**payload, 'timestamp': time.time(), 'nonce': uuid.uuid4().hex}
|
||||||
|
body = json.dumps(data, separators=(',', ':'), sort_keys=True).encode()
|
||||||
|
signature = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
|
||||||
|
return body, signature
|
||||||
|
|
||||||
|
|
||||||
|
def verify_request(body, signature):
|
||||||
|
expected = hmac.new(_secret(), body, hashlib.sha256).hexdigest()
|
||||||
|
if not signature or not hmac.compare_digest(expected, signature):
|
||||||
|
raise ValueError('模型服务认证失败')
|
||||||
|
data = json.loads(body)
|
||||||
|
if abs(time.time() - float(data.get('timestamp', 0))) > 60:
|
||||||
|
raise ValueError('模型服务请求已过期')
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_request(operation, model_id, **extra):
|
||||||
|
from maxkb.const import CONFIG
|
||||||
|
body, signature = signed_request({'operation': operation, 'model_id': str(model_id), **extra})
|
||||||
|
host = CONFIG.get('LOCAL_MODEL_HOST')
|
||||||
|
url = f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{host}:{CONFIG.get("LOCAL_MODEL_PORT")}{CONFIG.get_admin_path()}/api/model_runtime'
|
||||||
|
try:
|
||||||
|
response = requests.post(url, data=body, headers={'Content-Type': 'application/json',
|
||||||
|
'X-Model-Runtime-Signature': signature},
|
||||||
|
timeout=(5, 620 if operation == 'acquire' else 15))
|
||||||
|
result = response.json()
|
||||||
|
except (requests.RequestException, ValueError) as exc:
|
||||||
|
raise ValueError('本地模型服务不可用') from exc
|
||||||
|
if response.status_code != 200 or result.get('code') != 200:
|
||||||
|
raise ValueError(result.get('message') or '模型服务请求失败')
|
||||||
|
return result['data']
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def inference_lease(model_id):
|
||||||
|
if not model_id:
|
||||||
|
# Credential validation has no saved resource and must keep its original behavior.
|
||||||
|
yield None
|
||||||
|
return
|
||||||
|
data = runtime_request('acquire', model_id)
|
||||||
|
try:
|
||||||
|
yield data
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
runtime_request('release', model_id, lease=data['lease'])
|
||||||
|
except ValueError:
|
||||||
|
pass # Server expires abandoned leases; never hide the inference result.
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def async_inference_lease(model_id):
|
||||||
|
if not model_id:
|
||||||
|
yield None
|
||||||
|
return
|
||||||
|
data = await asyncio.to_thread(runtime_request, 'acquire', model_id)
|
||||||
|
try:
|
||||||
|
yield data
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(runtime_request, 'release', model_id, lease=data['lease'])
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
@ -274,16 +274,22 @@ def post_handler_paragraph(content: str, limit: int):
|
||||||
return reduce(lambda x, y: [*x, *y], map(lambda row: re.findall(pattern, row), result), [])
|
return reduce(lambda x, y: [*x, *y], map(lambda row: re.findall(pattern, row), result), [])
|
||||||
|
|
||||||
|
|
||||||
def smart_split_paragraph(content: str, limit: int):
|
def smart_split_paragraph(content: str, limit: int, overlap: int = 0):
|
||||||
"""
|
"""
|
||||||
智能分段:在limit前找到合适的分割点(句号、回车等)
|
智能分段:在limit前找到合适的分割点(句号、回车等)
|
||||||
:param content: 需要分段的文本
|
:param content: 需要分段的文本
|
||||||
:param limit: 最大字符限制
|
:param limit: 最大字符限制
|
||||||
|
:param overlap: 相邻分段的重叠长度
|
||||||
:return: 分段后的文本列表
|
:return: 分段后的文本列表
|
||||||
"""
|
"""
|
||||||
if len(content) <= limit:
|
if len(content) <= limit:
|
||||||
return [content]
|
return [content]
|
||||||
|
|
||||||
|
if overlap is None or overlap < 0:
|
||||||
|
overlap = 0
|
||||||
|
if overlap >= limit:
|
||||||
|
overlap = max(0, limit // 2)
|
||||||
|
|
||||||
result = []
|
result = []
|
||||||
start = 0
|
start = 0
|
||||||
|
|
||||||
|
|
@ -318,7 +324,9 @@ def smart_split_paragraph(content: str, limit: int):
|
||||||
best_split = end
|
best_split = end
|
||||||
|
|
||||||
result.append(content[start:best_split])
|
result.append(content[start:best_split])
|
||||||
start = best_split
|
# 相邻分段保留 overlap 字符重叠,且保证向前推进
|
||||||
|
next_start = best_split - overlap if overlap > 0 else best_split
|
||||||
|
start = next_start if next_start > start else best_split
|
||||||
|
|
||||||
return [text for text in result if text.strip()]
|
return [text for text in result if text.strip()]
|
||||||
|
|
||||||
|
|
@ -345,7 +353,7 @@ def filter_special_char(content: str):
|
||||||
|
|
||||||
class SplitModel:
|
class SplitModel:
|
||||||
|
|
||||||
def __init__(self, content_level_pattern, with_filter=True, limit=100000):
|
def __init__(self, content_level_pattern, with_filter=True, limit=100000, overlap=0):
|
||||||
self.content_level_pattern = content_level_pattern
|
self.content_level_pattern = content_level_pattern
|
||||||
self.with_filter = with_filter
|
self.with_filter = with_filter
|
||||||
if type(limit) is not int:
|
if type(limit) is not int:
|
||||||
|
|
@ -355,6 +363,16 @@ class SplitModel:
|
||||||
if limit < 50:
|
if limit < 50:
|
||||||
limit = 50
|
limit = 50
|
||||||
self.limit = limit
|
self.limit = limit
|
||||||
|
if type(overlap) is not int:
|
||||||
|
try:
|
||||||
|
overlap = int(overlap)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
overlap = 0
|
||||||
|
if overlap is None or overlap < 0:
|
||||||
|
overlap = 0
|
||||||
|
if overlap >= self.limit:
|
||||||
|
overlap = max(0, self.limit // 2)
|
||||||
|
self.overlap = overlap
|
||||||
|
|
||||||
def parse_to_tree(self, text: str, index=0):
|
def parse_to_tree(self, text: str, index=0):
|
||||||
"""
|
"""
|
||||||
|
|
@ -365,7 +383,8 @@ class SplitModel:
|
||||||
"""
|
"""
|
||||||
level_content_list = parse_title_level(text, self.content_level_pattern, index)
|
level_content_list = parse_title_level(text, self.content_level_pattern, index)
|
||||||
if len(level_content_list) == 0:
|
if len(level_content_list) == 0:
|
||||||
return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit)]
|
return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit,
|
||||||
|
overlap=self.overlap)]
|
||||||
if index == 0 and text.lstrip().index(level_content_list[0]["content"].lstrip()) != 0:
|
if index == 0 and text.lstrip().index(level_content_list[0]["content"].lstrip()) != 0:
|
||||||
level_content_list.insert(0, to_tree_obj(""))
|
level_content_list.insert(0, to_tree_obj(""))
|
||||||
|
|
||||||
|
|
@ -374,7 +393,7 @@ class SplitModel:
|
||||||
for i in range(len(level_title_content_list)):
|
for i in range(len(level_title_content_list)):
|
||||||
start_content: str = level_title_content_list[i].get('content')
|
start_content: str = level_title_content_list[i].get('content')
|
||||||
if cursor < text.index(start_content, cursor):
|
if cursor < text.index(start_content, cursor):
|
||||||
for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit):
|
for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit, overlap=self.overlap):
|
||||||
level_content_list.insert(0, to_tree_obj(row, 'block'))
|
level_content_list.insert(0, to_tree_obj(row, 'block'))
|
||||||
|
|
||||||
block, cursor = get_level_block(text, level_title_content_list, i, cursor)
|
block, cursor = get_level_block(text, level_title_content_list, i, cursor)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
"""按完整表格行分段;表头重复,重叠行不跨表/工作表。"""
|
||||||
|
def split_table_rows(header, rows, limit, overlap=0):
|
||||||
|
limit, overlap = int(limit), int(overlap)
|
||||||
|
if limit <= 0 or not 0 <= overlap < limit:
|
||||||
|
raise ValueError("Invalid table splitting limits")
|
||||||
|
chunks, current = [], []
|
||||||
|
length = len(header)
|
||||||
|
for row in rows:
|
||||||
|
if current and length + len(row) > limit:
|
||||||
|
chunks.append({"content": header + "".join(current), "title": ""})
|
||||||
|
carry, carried = [], 0
|
||||||
|
if overlap:
|
||||||
|
for previous in reversed(current):
|
||||||
|
carry.insert(0, previous)
|
||||||
|
carried += len(previous)
|
||||||
|
if carried >= overlap:
|
||||||
|
break
|
||||||
|
# 必须留出至少一条新行的空间,不能只重复旧行。
|
||||||
|
while carry and len(header) + sum(map(len, carry)) + len(row) > limit:
|
||||||
|
carry.pop(0)
|
||||||
|
current = carry
|
||||||
|
length = len(header) + sum(map(len, current))
|
||||||
|
current.append(row)
|
||||||
|
length += len(row)
|
||||||
|
if current:
|
||||||
|
chunks.append({"content": header + "".join(current), "title": ""})
|
||||||
|
return chunks
|
||||||
|
|
@ -16,7 +16,7 @@ from folders.api.folder import FolderCreateRequest
|
||||||
from knowledge.models import KnowledgeFolder, Knowledge
|
from knowledge.models import KnowledgeFolder, Knowledge
|
||||||
from knowledge.serializers.knowledge import KnowledgeSerializer
|
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||||||
from knowledge.serializers.knowledge_folder import KnowledgeFolderTreeSerializer
|
from knowledge.serializers.knowledge_folder import KnowledgeFolderTreeSerializer
|
||||||
from system_manage.models import WorkspaceUserResourcePermission
|
from system_manage.models import WorkspaceUserResourcePermission, EffectiveResourcePermission
|
||||||
from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer
|
from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer
|
||||||
from tools.models import ToolFolder, Tool
|
from tools.models import ToolFolder, Tool
|
||||||
from tools.serializers.tool import ToolSerializer
|
from tools.serializers.tool import ToolSerializer
|
||||||
|
|
@ -101,7 +101,7 @@ def get_max_depth(current_node):
|
||||||
|
|
||||||
|
|
||||||
def has_target_permission(workspace_id, source, user_id, target):
|
def has_target_permission(workspace_id, source, user_id, target):
|
||||||
return QuerySet(WorkspaceUserResourcePermission).filter(workspace_id=workspace_id, user_id=user_id,
|
return QuerySet(EffectiveResourcePermission).filter(workspace_id=workspace_id, user_id=user_id,
|
||||||
auth_target_type=source, target=target,
|
auth_target_type=source, target=target,
|
||||||
permission_list__contains=['MANAGE']).exists()
|
permission_list__contains=['MANAGE']).exists()
|
||||||
|
|
||||||
|
|
@ -245,17 +245,20 @@ class FolderSerializer(serializers.Serializer):
|
||||||
nodes = Folder.objects.filter(id=self.data.get('id')).get_descendants(include_self=True)
|
nodes = Folder.objects.filter(id=self.data.get('id')).get_descendants(include_self=True)
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
# 删除相关的资源
|
# 删除相关的资源
|
||||||
source_ids = (Source.objects.filter(folder_id=node.id)
|
source_query = Source.objects.filter(folder_id=node.id)
|
||||||
|
if Source is Knowledge:
|
||||||
|
source_query = source_query.exclude(meta__has_key='_version_snapshot')
|
||||||
|
source_ids = (source_query
|
||||||
.annotate(id_str=Cast('id', TextField()))
|
.annotate(id_str=Cast('id', TextField()))
|
||||||
.values_list('id_str', flat=True))
|
.values_list('id_str', flat=True))
|
||||||
# 检查文件夹是否存在未授权当前用户的资源
|
# 检查文件夹是否存在未授权当前用户的资源
|
||||||
auth_list = QuerySet(WorkspaceUserResourcePermission).filter(
|
auth_list = QuerySet(EffectiveResourcePermission).filter(
|
||||||
Q(workspace_id=self.data.get('workspace_id')) &
|
Q(workspace_id=self.data.get('workspace_id')) &
|
||||||
Q(user_id=self.data.get('user_id')) &
|
Q(user_id=self.data.get('user_id')) &
|
||||||
Q(auth_target_type=self.data.get('source')) &
|
Q(auth_target_type=self.data.get('source')) &
|
||||||
Q(target__in=source_ids) &
|
Q(target__in=source_ids) &
|
||||||
Q(permission_list__overlap=[ResourcePermission.MANAGE, ResourcePermissionRole.ROLE])
|
Q(permission_list__overlap=[ResourcePermission.MANAGE, ResourcePermissionRole.ROLE])
|
||||||
).count()
|
).values('target').distinct().count()
|
||||||
if auth_list != len(source_ids):
|
if auth_list != len(source_ids):
|
||||||
raise AppApiException(500, _('This folder contains resources that you dont have permission'))
|
raise AppApiException(500, _('This folder contains resources that you dont have permission'))
|
||||||
self.delete_source(node)
|
self.delete_source(node)
|
||||||
|
|
@ -263,7 +266,10 @@ class FolderSerializer(serializers.Serializer):
|
||||||
|
|
||||||
def delete_source(self, node):
|
def delete_source(self, node):
|
||||||
Source = get_source_type(self.data.get('source')) # noqa
|
Source = get_source_type(self.data.get('source')) # noqa
|
||||||
source_ids = Source.objects.filter(folder_id=node.id).values_list('id', flat=True)
|
source_query = Source.objects.filter(folder_id=node.id)
|
||||||
|
if Source is Knowledge:
|
||||||
|
source_query = source_query.exclude(meta__has_key='_version_snapshot')
|
||||||
|
source_ids = source_query.values_list('id', flat=True)
|
||||||
source = self.data.get('source')
|
source = self.data.get('source')
|
||||||
|
|
||||||
for source_id in source_ids:
|
for source_id in source_ids:
|
||||||
|
|
@ -319,7 +325,7 @@ class FolderTreeSerializer(serializers.Serializer):
|
||||||
if name is not None:
|
if name is not None:
|
||||||
base_q &= Q(name__contains=name)
|
base_q &= Q(name__contains=name)
|
||||||
if not workspace_manage:
|
if not workspace_manage:
|
||||||
base_q &= (Q(id__in=WorkspaceUserResourcePermission.objects.filter(user_id=current_user.id,
|
base_q &= (Q(id__in=EffectiveResourcePermission.objects.filter(user_id=current_user.id,
|
||||||
auth_target_type=self.data.get('source'),
|
auth_target_type=self.data.get('source'),
|
||||||
workspace_id=self.data.get(
|
workspace_id=self.data.get(
|
||||||
'workspace_id'),
|
'workspace_id'),
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,49 @@
|
||||||
|
"""Guard changes that would invalidate linked assistants or version snapshots."""
|
||||||
|
from django.db.models import OuterRef, Subquery
|
||||||
|
|
||||||
|
from application.models import Application, ApplicationKnowledgeMapping, ApplicationVersion
|
||||||
|
from knowledge.models import KnowledgeVersion
|
||||||
|
|
||||||
|
|
||||||
|
VERSION_REASON = '知识库已有历史版本,暂不支持更换嵌入模型或影响向量结果的参数。'
|
||||||
|
ASSISTANT_REASON = '关联助手使用了多个知识库,暂不支持更换嵌入模型或影响向量结果的参数。'
|
||||||
|
|
||||||
|
|
||||||
|
def flow_knowledge_ids(value):
|
||||||
|
"""Find static knowledge references, including nodes nested in loop bodies."""
|
||||||
|
ids = set()
|
||||||
|
if isinstance(value, dict):
|
||||||
|
if value.get('type') == 'search-knowledge-node':
|
||||||
|
data = (value.get('properties') or {}).get('node_data') or {}
|
||||||
|
ids.update(str(item) for item in (data.get('knowledge_id_list') or []) if item)
|
||||||
|
for child in value.values():
|
||||||
|
if isinstance(child, (dict, list)):
|
||||||
|
ids.update(flow_knowledge_ids(child))
|
||||||
|
elif isinstance(value, list):
|
||||||
|
for child in value:
|
||||||
|
if isinstance(child, (dict, list)):
|
||||||
|
ids.update(flow_knowledge_ids(child))
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def embedding_change_reason(knowledge):
|
||||||
|
if KnowledgeVersion.objects.filter(knowledge_id=knowledge.id).exists():
|
||||||
|
return VERSION_REASON
|
||||||
|
target = str(knowledge.id)
|
||||||
|
mappings = {}
|
||||||
|
for app_id, knowledge_id in ApplicationKnowledgeMapping.objects.values_list('application_id', 'knowledge_id'):
|
||||||
|
mappings.setdefault(str(app_id), set()).add(str(knowledge_id))
|
||||||
|
for app in Application.objects.only('id', 'type', 'work_flow').iterator():
|
||||||
|
ids = mappings.get(str(app.id), set()) | flow_knowledge_ids(app.work_flow)
|
||||||
|
if target in ids and len(ids) > 1:
|
||||||
|
return ASSISTANT_REASON
|
||||||
|
latest = ApplicationVersion.objects.filter(application_id=OuterRef('application_id')).order_by(
|
||||||
|
'-create_time', '-id').values('id')[:1]
|
||||||
|
for version in ApplicationVersion.objects.filter(id=Subquery(latest)).only(
|
||||||
|
'application_id', 'knowledge_setting', 'work_flow').iterator():
|
||||||
|
setting = version.knowledge_setting or {}
|
||||||
|
ids = {str(item) for item in setting.get('_knowledge_ids', mappings.get(str(version.application_id), set()))}
|
||||||
|
ids |= flow_knowledge_ids(version.work_flow)
|
||||||
|
if target in ids and len(ids) > 1:
|
||||||
|
return ASSISTANT_REASON
|
||||||
|
return ''
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
|
||||||
|
import math
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from langchain_core.embeddings import Embeddings
|
||||||
|
|
||||||
|
|
||||||
|
def validate_params(params):
|
||||||
|
if (not isinstance(params, dict) or len(params) > 52 or
|
||||||
|
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
|
||||||
|
raise ValueError('可选参数最多50项')
|
||||||
|
result, names = {}, set()
|
||||||
|
for key, value in params.items():
|
||||||
|
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
|
||||||
|
raise ValueError('参数名称须为1~64个字符')
|
||||||
|
key = key.strip()
|
||||||
|
normalized = unicodedata.normalize('NFKC', key).casefold()
|
||||||
|
if normalized in names:
|
||||||
|
raise ValueError('参数名称不能重复')
|
||||||
|
names.add(normalized)
|
||||||
|
if value is not None and (not isinstance(value, (str, int, float, bool)) or
|
||||||
|
isinstance(value, str) and len(value) > 2000 or
|
||||||
|
isinstance(value, float) and not math.isfinite(value)):
|
||||||
|
raise ValueError('参数值须为2000字以内的文本或数值')
|
||||||
|
result[key] = value
|
||||||
|
batch = result.get('batch_size')
|
||||||
|
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
|
||||||
|
raise ValueError('批处理数量须为1~256的整数')
|
||||||
|
if result.get('normalize') is not None and type(result['normalize']) is not bool:
|
||||||
|
raise ValueError('向量归一化参数无效')
|
||||||
|
for key in ('query_prefix', 'document_prefix'):
|
||||||
|
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
|
||||||
|
raise ValueError('向量指令须为2000字以内的文本')
|
||||||
|
return {key: value for key, value in result.items() if value is not None and
|
||||||
|
(value != '' or key not in {'query_prefix', 'document_prefix'})}
|
||||||
|
|
||||||
|
|
||||||
|
def knowledge_params(knowledge):
|
||||||
|
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
|
||||||
|
params.setdefault('normalize', True)
|
||||||
|
return {key: value for key, value in params.items()
|
||||||
|
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
|
||||||
|
|
||||||
|
|
||||||
|
class ConfiguredEmbedding(Embeddings):
|
||||||
|
def __init__(self, model, params):
|
||||||
|
self.model = model
|
||||||
|
self.params = params
|
||||||
|
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
|
||||||
|
|
||||||
|
def _vector(self, vector):
|
||||||
|
if not self.params.get('normalize', True):
|
||||||
|
return vector
|
||||||
|
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
|
||||||
|
return [float(v) / norm for v in vector] if norm else vector
|
||||||
|
|
||||||
|
def embed_query(self, text):
|
||||||
|
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
|
||||||
|
|
||||||
|
def embed_documents(self, texts):
|
||||||
|
prefix = self.params.get('document_prefix', '')
|
||||||
|
result = []
|
||||||
|
for start in range(0, len(texts), self.batch_size):
|
||||||
|
result.extend(self._vector(v) for v in self.model.embed_documents(
|
||||||
|
[prefix + text for text in texts[start:start + self.batch_size]]))
|
||||||
|
return result
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
# Generated for independent knowledge index rebuild task
|
||||||
|
import uuid_utils.compat
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('knowledge', '0003_tag_documenttag'),
|
||||||
|
('users', '0001_initial'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='KnowledgeRebuildTask',
|
||||||
|
fields=[
|
||||||
|
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
|
||||||
|
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
|
||||||
|
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
|
||||||
|
('workspace_id', models.CharField(db_index=True, default='default', max_length=64, verbose_name='工作空间id')),
|
||||||
|
('status', models.CharField(choices=[('PENDING', '待执行'), ('RUNNING', '执行中'), ('SUCCESS', '成功'), ('PARTIAL', '部分成功'), ('FAILED', '失败')], db_index=True, default='PENDING', max_length=20, verbose_name='任务状态')),
|
||||||
|
('config_snapshot', models.JSONField(default=dict, verbose_name='配置快照')),
|
||||||
|
('document_total', models.IntegerField(default=0, verbose_name='文档总数')),
|
||||||
|
('document_success', models.IntegerField(default=0, verbose_name='成功文档数')),
|
||||||
|
('document_failed', models.IntegerField(default=0, verbose_name='失败文档数')),
|
||||||
|
('error_message', models.TextField(blank=True, default='', verbose_name='失败原因')),
|
||||||
|
('started_at', models.DateTimeField(blank=True, null=True, verbose_name='开始时间')),
|
||||||
|
('finished_at', models.DateTimeField(blank=True, null=True, verbose_name='结束时间')),
|
||||||
|
('celery_task_id', models.CharField(blank=True, default='', max_length=128, verbose_name='Celery任务id')),
|
||||||
|
('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, related_name='rebuild_tasks', to='knowledge.knowledge', verbose_name='知识库')),
|
||||||
|
('operator', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='users.user', verbose_name='操作人')),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'db_table': 'knowledge_rebuild_task',
|
||||||
|
'indexes': [models.Index(fields=['knowledge', 'status'], name='knowledge_r_knowled_0a6e2f_idx')],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Generated for rebuild task attempt polling
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('knowledge', '0004_knowledge_rebuild_task'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledgerebuildtask',
|
||||||
|
name='attempt',
|
||||||
|
field=models.IntegerField(default=0, verbose_name='汇总轮询次数'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,82 @@
|
||||||
|
# Generated by Django 5.2.8 on 2026-10-27 03:14
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid_utils.compat
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('knowledge', '0005_rebuild_task_attempt'),
|
||||||
|
('users', '0002_user_activation_and_role_assignment'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledge',
|
||||||
|
name='version_sequence',
|
||||||
|
field=models.PositiveIntegerField(default=0, editable=False),
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='KnowledgeVersion',
|
||||||
|
fields=[
|
||||||
|
('create_time', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='创建时间')),
|
||||||
|
('update_time', models.DateTimeField(auto_now=True, db_index=True, verbose_name='修改时间')),
|
||||||
|
('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False)),
|
||||||
|
('number', models.PositiveIntegerField()),
|
||||||
|
('name', models.CharField(max_length=128)),
|
||||||
|
('description', models.CharField(blank=True, default='', max_length=512)),
|
||||||
|
('creator_name', models.CharField(default='', max_length=128)),
|
||||||
|
('document_count', models.PositiveIntegerField(default=0)),
|
||||||
|
('paragraph_count', models.PositiveIntegerField(default=0)),
|
||||||
|
('vector_count', models.PositiveIntegerField(default=0)),
|
||||||
|
('search_ready', models.BooleanField(default=False)),
|
||||||
|
('content_hash', models.CharField(max_length=64)),
|
||||||
|
('config', models.JSONField(default=dict)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'db_table': 'knowledge_version',
|
||||||
|
'ordering': ['-number'],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='KnowledgeVersionLease',
|
||||||
|
fields=[
|
||||||
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||||
|
('chat_id', models.CharField(max_length=128)),
|
||||||
|
('expires_at', models.DateTimeField(db_index=True)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'db_table': 'knowledge_version_lease',
|
||||||
|
},
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledgeversion',
|
||||||
|
name='creator',
|
||||||
|
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='users.user'),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledgeversion',
|
||||||
|
name='knowledge',
|
||||||
|
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='versions', to='knowledge.knowledge'),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledgeversion',
|
||||||
|
name='snapshot',
|
||||||
|
field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='knowledge.knowledge'),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name='knowledgeversionlease',
|
||||||
|
name='version',
|
||||||
|
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='leases', to='knowledge.knowledgeversion'),
|
||||||
|
),
|
||||||
|
migrations.AddConstraint(
|
||||||
|
model_name='knowledgeversion',
|
||||||
|
constraint=models.UniqueConstraint(fields=('knowledge', 'number'), name='knowledge_version_number_unique'),
|
||||||
|
),
|
||||||
|
migrations.AddConstraint(
|
||||||
|
model_name='knowledgeversionlease',
|
||||||
|
constraint=models.UniqueConstraint(fields=('version', 'chat_id'), name='knowledge_version_chat_unique'),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
@ -1 +1,2 @@
|
||||||
from .knowledge import *
|
from .knowledge import *
|
||||||
|
from .version import KnowledgeVersion, KnowledgeVersionLease
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,46 @@ class KnowledgeScope(models.TextChoices):
|
||||||
WORKSPACE = "WORKSPACE", "工作空间可用"
|
WORKSPACE = "WORKSPACE", "工作空间可用"
|
||||||
|
|
||||||
|
|
||||||
|
class RebuildStatus(models.TextChoices):
|
||||||
|
PENDING = "PENDING", "待执行"
|
||||||
|
RUNNING = "RUNNING", "执行中"
|
||||||
|
SUCCESS = "SUCCESS", "成功"
|
||||||
|
PARTIAL = "PARTIAL", "部分成功"
|
||||||
|
FAILED = "FAILED", "失败"
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeRebuildTask(AppModelMixin):
|
||||||
|
"""知识库独立索引重建任务。
|
||||||
|
|
||||||
|
与“更换向量模型自动触发重向量化”分开:本表记录管理员在配置不变时
|
||||||
|
主动发起的重建请求、执行状态、文档统计和失败原因。
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id")
|
||||||
|
knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.CASCADE, db_constraint=False,
|
||||||
|
related_name="rebuild_tasks", verbose_name="知识库")
|
||||||
|
workspace_id = models.CharField(max_length=64, verbose_name="工作空间id", default="default", db_index=True)
|
||||||
|
status = models.CharField(max_length=20, choices=RebuildStatus.choices, default=RebuildStatus.PENDING,
|
||||||
|
db_index=True, verbose_name="任务状态")
|
||||||
|
config_snapshot = models.JSONField(verbose_name="配置快照", default=dict)
|
||||||
|
document_total = models.IntegerField(verbose_name="文档总数", default=0)
|
||||||
|
document_success = models.IntegerField(verbose_name="成功文档数", default=0)
|
||||||
|
document_failed = models.IntegerField(verbose_name="失败文档数", default=0)
|
||||||
|
error_message = models.TextField(verbose_name="失败原因", default="", blank=True)
|
||||||
|
operator = models.ForeignKey(User, on_delete=models.SET_NULL, db_constraint=False, null=True, blank=True,
|
||||||
|
related_name="+", verbose_name="操作人")
|
||||||
|
started_at = models.DateTimeField(verbose_name="开始时间", null=True, blank=True)
|
||||||
|
finished_at = models.DateTimeField(verbose_name="结束时间", null=True, blank=True)
|
||||||
|
celery_task_id = models.CharField(max_length=128, verbose_name="Celery任务id", default="", blank=True)
|
||||||
|
attempt = models.IntegerField(verbose_name="汇总轮询次数", default=0)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
db_table = "knowledge_rebuild_task"
|
||||||
|
indexes = [
|
||||||
|
models.Index(fields=["knowledge", "status"], name='knowledge_r_knowled_0a6e2f_idx'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
class HitHandlingMethod(models.TextChoices):
|
class HitHandlingMethod(models.TextChoices):
|
||||||
optimization = 'optimization', '模型优化'
|
optimization = 'optimization', '模型优化'
|
||||||
directly_return = 'directly_return', '直接返回'
|
directly_return = 'directly_return', '直接返回'
|
||||||
|
|
@ -130,6 +170,7 @@ class Knowledge(AppModelMixin):
|
||||||
file_size_limit = models.IntegerField(verbose_name="文件大小限制", default=100)
|
file_size_limit = models.IntegerField(verbose_name="文件大小限制", default=100)
|
||||||
file_count_limit = models.IntegerField(verbose_name="文件数量限制", default=50)
|
file_count_limit = models.IntegerField(verbose_name="文件数量限制", default=50)
|
||||||
meta = models.JSONField(verbose_name="元数据", default=dict)
|
meta = models.JSONField(verbose_name="元数据", default=dict)
|
||||||
|
version_sequence = models.PositiveIntegerField(default=0, editable=False)
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
db_table = "knowledge"
|
db_table = "knowledge"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,36 @@
|
||||||
|
"""Immutable knowledge releases and short-lived conversation references."""
|
||||||
|
import uuid_utils.compat as uuid
|
||||||
|
from django.db import models
|
||||||
|
from common.mixins.app_model_mixin import AppModelMixin
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeVersion(AppModelMixin):
|
||||||
|
id = models.UUIDField(primary_key=True, default=uuid.uuid7, editable=False)
|
||||||
|
knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.PROTECT, related_name='versions')
|
||||||
|
snapshot = models.OneToOneField('knowledge.Knowledge', on_delete=models.PROTECT, related_name='+')
|
||||||
|
number = models.PositiveIntegerField()
|
||||||
|
name = models.CharField(max_length=128)
|
||||||
|
description = models.CharField(max_length=512, default='', blank=True)
|
||||||
|
creator = models.ForeignKey('users.User', null=True, on_delete=models.SET_NULL)
|
||||||
|
creator_name = models.CharField(max_length=128, default='')
|
||||||
|
document_count = models.PositiveIntegerField(default=0)
|
||||||
|
paragraph_count = models.PositiveIntegerField(default=0)
|
||||||
|
vector_count = models.PositiveIntegerField(default=0)
|
||||||
|
search_ready = models.BooleanField(default=False)
|
||||||
|
content_hash = models.CharField(max_length=64)
|
||||||
|
config = models.JSONField(default=dict)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
db_table = 'knowledge_version'
|
||||||
|
ordering = ['-number']
|
||||||
|
constraints = [models.UniqueConstraint(fields=['knowledge', 'number'], name='knowledge_version_number_unique')]
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeVersionLease(models.Model):
|
||||||
|
version = models.ForeignKey(KnowledgeVersion, on_delete=models.CASCADE, related_name='leases')
|
||||||
|
chat_id = models.CharField(max_length=128)
|
||||||
|
expires_at = models.DateTimeField(db_index=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
db_table = 'knowledge_version_lease'
|
||||||
|
constraints = [models.UniqueConstraint(fields=['version', 'chat_id'], name='knowledge_version_chat_unique')]
|
||||||
|
|
@ -114,40 +114,31 @@ class ProblemParagraphManage:
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model_by_knowledge_id_list(knowledge_id_list: List):
|
def get_embedding_model_by_knowledge_id_list(knowledge_id_list: List):
|
||||||
knowledge_list = QuerySet(Knowledge).filter(id__in=knowledge_id_list)
|
knowledge_list = list(QuerySet(Knowledge).select_related('embedding_model').filter(id__in=knowledge_id_list))
|
||||||
if len(set([knowledge.embedding_model_id for knowledge in knowledge_list])) > 1:
|
if not knowledge_list:
|
||||||
raise Exception(_('The knowledge base is inconsistent with the vector model'))
|
|
||||||
if len(knowledge_list) == 0:
|
|
||||||
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
||||||
if knowledge_list[0].embedding_model_id is None or knowledge_list[0].embedding_model is None:
|
from knowledge.embedding_params import knowledge_params
|
||||||
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
configs = [(str(k.embedding_model_id), {key: value for key, value in knowledge_params(k).items()
|
||||||
|
if key != 'batch_size'}) for k in knowledge_list]
|
||||||
default_params = get_model_default_params(knowledge_list[0].embedding_model)
|
if any(config != configs[0] for config in configs[1:]):
|
||||||
|
raise Exception('知识库的向量模型或嵌入参数不一致')
|
||||||
return ModelManage.get_model(
|
return get_embedding_model_by_knowledge(knowledge_list[0])
|
||||||
str(knowledge_list[0].embedding_model_id),
|
|
||||||
lambda _id: get_model(knowledge_list[0].embedding_model, **{**default_params})
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model_by_knowledge_id(knowledge_id: str):
|
def get_embedding_model_by_knowledge_id(knowledge_id: str):
|
||||||
knowledge = QuerySet(Knowledge).select_related('embedding_model').filter(id=knowledge_id).first()
|
knowledge = QuerySet(Knowledge).select_related('embedding_model').filter(id=knowledge_id).first()
|
||||||
if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None:
|
return get_embedding_model_by_knowledge(knowledge)
|
||||||
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
|
||||||
|
|
||||||
default_params = get_model_default_params(knowledge.embedding_model)
|
|
||||||
|
|
||||||
return ModelManage.get_model(str(knowledge.embedding_model_id),
|
|
||||||
lambda _id: get_model(knowledge.embedding_model, **{**default_params}))
|
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model_by_knowledge(knowledge):
|
def get_embedding_model_by_knowledge(knowledge):
|
||||||
if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None:
|
if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None:
|
||||||
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
raise Exception(_('Knowledge base setting error, please reset the knowledge base'))
|
||||||
|
from knowledge.embedding_params import ConfiguredEmbedding, knowledge_params
|
||||||
default_params = get_model_default_params(knowledge.embedding_model)
|
default_params = get_model_default_params(knowledge.embedding_model)
|
||||||
|
model = ModelManage.get_model(str(knowledge.embedding_model_id),
|
||||||
return ModelManage.get_model(str(knowledge.embedding_model_id),
|
lambda _id: get_model(knowledge.embedding_model, **default_params))
|
||||||
lambda _id: get_model(knowledge.embedding_model, **{**default_params}))
|
params = knowledge_params(knowledge)
|
||||||
|
return ConfiguredEmbedding(model, params) if params else model
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model_id_by_knowledge_id(knowledge_id):
|
def get_embedding_model_id_by_knowledge_id(knowledge_id):
|
||||||
|
|
|
||||||
|
|
@ -162,7 +162,8 @@ class DocumentEditInstanceSerializer(serializers.Serializer):
|
||||||
|
|
||||||
class DocumentSplitRequest(serializers.Serializer):
|
class DocumentSplitRequest(serializers.Serializer):
|
||||||
file = serializers.ListField(required=True, label=_('file list'))
|
file = serializers.ListField(required=True, label=_('file list'))
|
||||||
limit = serializers.IntegerField(required=False, label=_('limit'))
|
limit = serializers.IntegerField(required=False, min_value=50, max_value=100000, label=_('limit'))
|
||||||
|
overlap = serializers.IntegerField(required=False, min_value=0, label=_('chunk overlap'))
|
||||||
patterns = serializers.ListField(
|
patterns = serializers.ListField(
|
||||||
required=False,
|
required=False,
|
||||||
child=serializers.CharField(required=True, label=_('patterns')),
|
child=serializers.CharField(required=True, label=_('patterns')),
|
||||||
|
|
@ -338,7 +339,8 @@ class DocumentSerializers(serializers.Serializer):
|
||||||
else:
|
else:
|
||||||
document_list.update(knowledge_id=target_knowledge_id)
|
document_list.update(knowledge_id=target_knowledge_id)
|
||||||
model_id = None
|
model_id = None
|
||||||
if knowledge.embedding_model_id != target_knowledge.embedding_model_id:
|
if (knowledge.embedding_model_id != target_knowledge.embedding_model_id or
|
||||||
|
(knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})):
|
||||||
model_id = get_embedding_model_id_by_knowledge_id(target_knowledge_id)
|
model_id = get_embedding_model_id_by_knowledge_id(target_knowledge_id)
|
||||||
|
|
||||||
pid_list = [paragraph.id for paragraph in paragraph_list]
|
pid_list = [paragraph.id for paragraph in paragraph_list]
|
||||||
|
|
@ -1047,16 +1049,38 @@ class DocumentSerializers(serializers.Serializer):
|
||||||
|
|
||||||
def parse(self, instance):
|
def parse(self, instance):
|
||||||
self.is_valid(instance=instance, raise_exception=True)
|
self.is_valid(instance=instance, raise_exception=True)
|
||||||
DocumentSplitRequest(data=instance).is_valid(raise_exception=True)
|
split_request = DocumentSplitRequest(data=instance)
|
||||||
|
split_request.is_valid(raise_exception=True)
|
||||||
|
instance = split_request.validated_data
|
||||||
|
|
||||||
|
# 分段长度/重叠未显式指定时,回退到知识库分段规则(meta.segmentation)
|
||||||
|
knowledge = QuerySet(Knowledge).filter(id=self.data.get('knowledge_id')).first()
|
||||||
|
segmentation = (knowledge.meta or {}).get('segmentation') if knowledge is not None else None
|
||||||
|
segmentation = segmentation or {}
|
||||||
|
default_limit = segmentation.get('chunk_size') or 500
|
||||||
|
default_overlap = segmentation.get('chunk_overlap') or 0
|
||||||
|
limit = instance.get("limit", None) or default_limit
|
||||||
|
overlap = instance.get("overlap", None)
|
||||||
|
overlap = default_overlap if overlap is None else overlap
|
||||||
|
if not 0 <= overlap < limit:
|
||||||
|
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
|
||||||
|
|
||||||
file_list = instance.get("file")
|
file_list = instance.get("file")
|
||||||
|
from system_manage.access_control import enforce
|
||||||
|
from system_manage.parsers import extension, MINERU_INPUTS
|
||||||
|
request = self.context.get('request')
|
||||||
|
enforce(request, 'knowledge', self.data.get('knowledge_id'))
|
||||||
|
parser_id = (knowledge.meta or {}).get('parser_id') if knowledge else None
|
||||||
|
if parser_id and any(extension(file.name) in MINERU_INPUTS for file in file_list):
|
||||||
|
enforce(request, 'parser', parser_id)
|
||||||
return reduce(
|
return reduce(
|
||||||
lambda x, y: [*x, *y],
|
lambda x, y: [*x, *y],
|
||||||
[self.file_to_paragraph(
|
[self.file_to_paragraph(
|
||||||
f,
|
f,
|
||||||
instance.get("patterns", None),
|
instance.get("patterns", None),
|
||||||
instance.get("with_filter", None),
|
instance.get("with_filter", None),
|
||||||
instance.get("limit", 4096)
|
limit,
|
||||||
|
overlap
|
||||||
) for f in file_list],
|
) for f in file_list],
|
||||||
[]
|
[]
|
||||||
)
|
)
|
||||||
|
|
@ -1095,7 +1119,7 @@ class DocumentSerializers(serializers.Serializer):
|
||||||
item['source_file_id'] = file_id
|
item['source_file_id'] = file_id
|
||||||
return result_list
|
return result_list
|
||||||
|
|
||||||
def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int):
|
def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int, overlap: int = 0):
|
||||||
# 保存源文件
|
# 保存源文件
|
||||||
file_id = uuid.uuid7()
|
file_id = uuid.uuid7()
|
||||||
raw_file = File(
|
raw_file = File(
|
||||||
|
|
@ -1108,12 +1132,22 @@ class DocumentSerializers(serializers.Serializer):
|
||||||
raw_file.save(file.read())
|
raw_file.save(file.read())
|
||||||
file.seek(0)
|
file.seek(0)
|
||||||
|
|
||||||
|
from system_manage.ai_hub_knowledge import split_with_parser
|
||||||
|
parsed = split_with_parser(self.data.get('workspace_id'), self.data.get('knowledge_id'),
|
||||||
|
self.context.get('user'), file, pattern_list, with_filter, limit, overlap)
|
||||||
|
if parsed is not None:
|
||||||
|
result, provenance = parsed
|
||||||
|
raw_file.meta = {**(raw_file.meta or {}), 'parser': provenance}
|
||||||
|
QuerySet(File).filter(id=file_id).update(meta=raw_file.meta)
|
||||||
|
return self.normalize_split_result(result, file, default_split_handle, file_id)
|
||||||
get_buffer = FileBufferHandle().get_buffer
|
get_buffer = FileBufferHandle().get_buffer
|
||||||
for split_handle in split_handles:
|
for split_handle in split_handles:
|
||||||
if split_handle.support(file, get_buffer):
|
if split_handle.support(file, get_buffer):
|
||||||
result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image)
|
result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image,
|
||||||
|
overlap)
|
||||||
return self.normalize_split_result(result, file, split_handle, file_id)
|
return self.normalize_split_result(result, file, split_handle, file_id)
|
||||||
result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image)
|
result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image,
|
||||||
|
overlap)
|
||||||
return self.normalize_split_result(result, file, default_split_handle, file_id)
|
return self.normalize_split_result(result, file, default_split_handle, file_id)
|
||||||
|
|
||||||
class SplitPattern(serializers.Serializer):
|
class SplitPattern(serializers.Serializer):
|
||||||
|
|
@ -1706,7 +1740,7 @@ class DocumentSerializers(serializers.Serializer):
|
||||||
data={'workspace_id': self.data.get('workspace_id'), 'knowledge_id': self.data.get('knowledge_id')}
|
data={'workspace_id': self.data.get('workspace_id'), 'knowledge_id': self.data.get('knowledge_id')}
|
||||||
)
|
)
|
||||||
split_serializer.is_valid(instance={'file': [file]}, raise_exception=True)
|
split_serializer.is_valid(instance={'file': [file]}, raise_exception=True)
|
||||||
parsed_documents = split_serializer.file_to_paragraph(file, None, None, 4096)
|
parsed_documents = split_serializer.file_to_paragraph(file, None, None, 500)
|
||||||
paragraphs = []
|
paragraphs = []
|
||||||
for parsed_document in parsed_documents:
|
for parsed_document in parsed_documents:
|
||||||
paragraphs.extend(parsed_document.get('content', []))
|
paragraphs.extend(parsed_document.get('content', []))
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,7 @@ class KnowledgeWebCreateRequest(serializers.Serializer):
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeEditRequest(serializers.Serializer):
|
class KnowledgeEditRequest(serializers.Serializer):
|
||||||
|
embedding_params = serializers.DictField(required=False)
|
||||||
name = serializers.CharField(required=False, max_length=64, min_length=1, label=_('knowledge name'))
|
name = serializers.CharField(required=False, max_length=64, min_length=1, label=_('knowledge name'))
|
||||||
desc = serializers.CharField(required=False, max_length=256, min_length=1, label=_('knowledge description'))
|
desc = serializers.CharField(required=False, max_length=256, min_length=1, label=_('knowledge description'))
|
||||||
meta = serializers.DictField(required=False)
|
meta = serializers.DictField(required=False)
|
||||||
|
|
@ -166,7 +167,8 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
|
|
||||||
query_set_dict['knowledge_custom_sql'] = QuerySet(model=get_dynamics_model({
|
query_set_dict['knowledge_custom_sql'] = QuerySet(model=get_dynamics_model({
|
||||||
'knowledge.workspace_id': models.CharField(),
|
'knowledge.workspace_id': models.CharField(),
|
||||||
})).filter(**{'knowledge.workspace_id': workspace_id})
|
'knowledge.meta': models.JSONField(),
|
||||||
|
})).filter(**{'knowledge.workspace_id': workspace_id}).exclude(**{'knowledge.meta__has_key': '_version_snapshot'})
|
||||||
# query_set_dict['folder_query_set'] = folder_query_set
|
# query_set_dict['folder_query_set'] = folder_query_set
|
||||||
if not workspace_manage:
|
if not workspace_manage:
|
||||||
query_set_dict['workspace_user_resource_permission_query_set'] = QuerySet(
|
query_set_dict['workspace_user_resource_permission_query_set'] = QuerySet(
|
||||||
|
|
@ -344,9 +346,12 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
), with_search_one=True)
|
), with_search_one=True)
|
||||||
|
from knowledge.embedding_guard import embedding_change_reason
|
||||||
return {
|
return {
|
||||||
**knowledge_dict,
|
**knowledge_dict,
|
||||||
'meta': json.loads(knowledge_dict.get('meta', '{}')),
|
'meta': json.loads(knowledge_dict.get('meta', '{}')),
|
||||||
|
'embedding_change_block_reason': embedding_change_reason(
|
||||||
|
QuerySet(Knowledge).get(id=self.data.get('knowledge_id'))),
|
||||||
'application_id_list': list(filter(
|
'application_id_list': list(filter(
|
||||||
lambda application_id: all_application_list.__contains__(application_id),
|
lambda application_id: all_application_list.__contains__(application_id),
|
||||||
[
|
[
|
||||||
|
|
@ -361,7 +366,7 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
@transaction.atomic
|
@transaction.atomic
|
||||||
def edit(self, instance: Dict, select_one=True):
|
def edit(self, instance: Dict, select_one=True):
|
||||||
self.is_valid()
|
self.is_valid()
|
||||||
knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id"))
|
knowledge = QuerySet(Knowledge).select_for_update().get(id=self.data.get("knowledge_id"))
|
||||||
if QuerySet(Knowledge).filter(
|
if QuerySet(Knowledge).filter(
|
||||||
workspace_id=knowledge.workspace_id,
|
workspace_id=knowledge.workspace_id,
|
||||||
name=instance.get('name'),
|
name=instance.get('name'),
|
||||||
|
|
@ -369,6 +374,23 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
).exclude(id=knowledge.id).exists():
|
).exclude(id=knowledge.id).exists():
|
||||||
raise AppApiException(500, _('Knowledge base name duplicate!'))
|
raise AppApiException(500, _('Knowledge base name duplicate!'))
|
||||||
KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge)
|
KnowledgeEditRequest(data=instance).is_valid(knowledge=knowledge)
|
||||||
|
from knowledge.embedding_params import knowledge_params, validate_params
|
||||||
|
from knowledge.embedding_guard import embedding_change_reason
|
||||||
|
from types import SimpleNamespace
|
||||||
|
current_params = knowledge_params(knowledge)
|
||||||
|
try:
|
||||||
|
params = validate_params(instance['embedding_params']) if 'embedding_params' in instance else None
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AppApiException(500, str(exc))
|
||||||
|
new_params = knowledge_params(SimpleNamespace(meta={'embedding_params': params})) if params is not None else current_params
|
||||||
|
model_changed = ('embedding_model_id' in instance and
|
||||||
|
str(instance['embedding_model_id'] or '') != str(knowledge.embedding_model_id or ''))
|
||||||
|
vector_params_changed = ({k: v for k, v in current_params.items() if k != 'batch_size'} !=
|
||||||
|
{k: v for k, v in new_params.items() if k != 'batch_size'})
|
||||||
|
if model_changed or vector_params_changed:
|
||||||
|
reason = embedding_change_reason(knowledge)
|
||||||
|
if reason:
|
||||||
|
raise AppApiException(500, reason)
|
||||||
if 'embedding_model_id' in instance:
|
if 'embedding_model_id' in instance:
|
||||||
knowledge.embedding_model_id = instance.get('embedding_model_id')
|
knowledge.embedding_model_id = instance.get('embedding_model_id')
|
||||||
if "name" in instance:
|
if "name" in instance:
|
||||||
|
|
@ -376,7 +398,15 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
if 'desc' in instance:
|
if 'desc' in instance:
|
||||||
knowledge.desc = instance.get("desc")
|
knowledge.desc = instance.get("desc")
|
||||||
if 'meta' in instance:
|
if 'meta' in instance:
|
||||||
knowledge.meta = instance.get('meta')
|
incoming_meta = dict(instance.get('meta') or {})
|
||||||
|
# Independently saved metadata/rules must survive stale general settings forms.
|
||||||
|
for managed_key in ('business_metadata', 'segmentation', 'parser_id', 'embedding_params'):
|
||||||
|
incoming_meta.pop(managed_key, None)
|
||||||
|
if managed_key in (knowledge.meta or {}):
|
||||||
|
incoming_meta[managed_key] = knowledge.meta[managed_key]
|
||||||
|
knowledge.meta = incoming_meta
|
||||||
|
if 'embedding_params' in instance:
|
||||||
|
knowledge.meta = {**(knowledge.meta or {}), 'embedding_params': params}
|
||||||
if 'folder_id' in instance:
|
if 'folder_id' in instance:
|
||||||
knowledge.folder_id = instance.get('folder_id')
|
knowledge.folder_id = instance.get('folder_id')
|
||||||
if 'file_size_limit' in instance:
|
if 'file_size_limit' in instance:
|
||||||
|
|
@ -384,6 +414,10 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
if 'file_count_limit' in instance:
|
if 'file_count_limit' in instance:
|
||||||
knowledge.file_count_limit = instance.get('file_count_limit')
|
knowledge.file_count_limit = instance.get('file_count_limit')
|
||||||
if 'user_id' in instance:
|
if 'user_id' in instance:
|
||||||
|
if str(instance.get('user_id')) != str(knowledge.user_id):
|
||||||
|
from knowledge.serializers.knowledge_ext import _creator_candidates
|
||||||
|
if not _creator_candidates(knowledge.workspace_id).filter(id=instance.get('user_id')).exists():
|
||||||
|
raise AppApiException(500, '创建者须为已激活且启用的成员')
|
||||||
knowledge.user_id = instance.get('user_id')
|
knowledge.user_id = instance.get('user_id')
|
||||||
if 'create_user' in instance:
|
if 'create_user' in instance:
|
||||||
knowledge.create_user = instance.get('create_user')
|
knowledge.create_user = instance.get('create_user')
|
||||||
|
|
@ -422,6 +456,9 @@ class KnowledgeSerializer(serializers.Serializer):
|
||||||
def delete(self):
|
def delete(self):
|
||||||
self.is_valid()
|
self.is_valid()
|
||||||
knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id"))
|
knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id"))
|
||||||
|
from knowledge.models import KnowledgeVersion
|
||||||
|
if KnowledgeVersion.objects.filter(knowledge=knowledge).exists():
|
||||||
|
raise AppApiException(500, '请先删除该知识库的历史版本')
|
||||||
QuerySet(Document).filter(knowledge=knowledge).delete()
|
QuerySet(Document).filter(knowledge=knowledge).delete()
|
||||||
QuerySet(ProblemParagraphMapping).filter(knowledge=knowledge).delete()
|
QuerySet(ProblemParagraphMapping).filter(knowledge=knowledge).delete()
|
||||||
QuerySet(Paragraph).filter(knowledge=knowledge).delete()
|
QuerySet(Paragraph).filter(knowledge=knowledge).delete()
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,215 @@
|
||||||
|
# coding=utf-8
|
||||||
|
"""知识库扩展配置序列化器:业务元数据、分段规则。"""
|
||||||
|
from collections.abc import Mapping
|
||||||
|
import re
|
||||||
|
|
||||||
|
from django.db import transaction
|
||||||
|
from django.db.models import QuerySet
|
||||||
|
from django.utils import timezone
|
||||||
|
from rest_framework import serializers
|
||||||
|
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
from knowledge.models import Knowledge
|
||||||
|
|
||||||
|
METADATA_FIELDS = [
|
||||||
|
{"field": "business_domain", "label": "业务领域"},
|
||||||
|
{"field": "data_source", "label": "数据来源"},
|
||||||
|
{"field": "owner_dept", "label": "责任部门"},
|
||||||
|
{"field": "business_version", "label": "业务版本"},
|
||||||
|
]
|
||||||
|
METADATA_MAX_LEN = 64
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt(value):
|
||||||
|
if not value:
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
value = timezone.localtime(value)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_knowledge(workspace_id: str, knowledge_id: str, lock=False) -> Knowledge:
|
||||||
|
query = QuerySet(Knowledge)
|
||||||
|
if lock:
|
||||||
|
query = query.select_for_update()
|
||||||
|
knowledge = query.filter(id=knowledge_id, workspace_id=workspace_id).first()
|
||||||
|
if knowledge is None:
|
||||||
|
raise AppApiException(500, "知识库不存在")
|
||||||
|
return knowledge
|
||||||
|
|
||||||
|
|
||||||
|
def _creator_candidates(workspace_id):
|
||||||
|
from users.models import User, UserActivationApplication
|
||||||
|
from common.database_model_manage.database_model_manage import DatabaseModelManage
|
||||||
|
from django.db.models import Q
|
||||||
|
pending = UserActivationApplication.objects.exclude(status='APPROVED').values_list('user_id', flat=True)
|
||||||
|
query = User.objects.filter(is_active=True).exclude(id__in=pending)
|
||||||
|
membership = DatabaseModelManage.get_model('workspace_user_role_mapping')
|
||||||
|
if membership is not None:
|
||||||
|
ids = membership.objects.filter(workspace_id=workspace_id).values_list('user_id', flat=True)
|
||||||
|
query = query.filter(Q(id__in=ids) | Q(role='ADMIN'))
|
||||||
|
return query
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata_fields(business):
|
||||||
|
# An explicit empty list means all custom fields were removed; do not revive old fields.
|
||||||
|
if isinstance(business.get('custom_fields'), list):
|
||||||
|
return business['custom_fields']
|
||||||
|
return [{'name': item['label'], 'value': business[item['field']]}
|
||||||
|
for item in METADATA_FIELDS if business.get(item['field']) not in (None, '')]
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_custom_fields(items):
|
||||||
|
import unicodedata
|
||||||
|
if not isinstance(items, list) or len(items) > 50:
|
||||||
|
raise AppApiException(500, '元数据最多50项')
|
||||||
|
reserved = {'创建者', '创建时间', '更新时间', 'creator', 'user_id', 'create_time', 'update_time', 'updated_at', 'updated_by'}
|
||||||
|
result, names = [], set()
|
||||||
|
for item in items:
|
||||||
|
if not isinstance(item, Mapping) or set(item) != {'name', 'value'}:
|
||||||
|
raise AppApiException(500, '元数据需包含名称和值')
|
||||||
|
name, value = item['name'], item['value']
|
||||||
|
if not isinstance(name, str) or not name.strip() or len(name.strip()) > 64 or any(ord(c) < 32 for c in name):
|
||||||
|
raise AppApiException(500, '元数据名称须为1~64个字符')
|
||||||
|
name = name.strip()
|
||||||
|
normalized = unicodedata.normalize('NFKC', name).casefold()
|
||||||
|
if normalized in reserved:
|
||||||
|
raise AppApiException(500, '不能使用固定属性名称')
|
||||||
|
if normalized in names:
|
||||||
|
raise AppApiException(500, '元数据名称不能重复')
|
||||||
|
if not isinstance(value, str) or len(value) > 2000:
|
||||||
|
raise AppApiException(500, '元数据值须为2000字以内的文本')
|
||||||
|
names.add(normalized)
|
||||||
|
result.append({'name': name, 'value': value.strip()})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeMetadataSerializer(serializers.Serializer):
|
||||||
|
"""Fixed system attributes plus freely named business metadata; save atomically."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def one(workspace_id: str, knowledge_id: str):
|
||||||
|
from users.models import User
|
||||||
|
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||||||
|
data = (knowledge.meta or {}).get('business_metadata') or {}
|
||||||
|
creator = User.objects.filter(id=knowledge.user_id).first() if knowledge.user_id else None
|
||||||
|
candidates = [{'id': str(u.id), 'nick_name': u.nick_name or u.username, 'disabled': False}
|
||||||
|
for u in _creator_candidates(workspace_id).order_by('nick_name')]
|
||||||
|
if creator and not any(u['id'] == str(creator.id) for u in candidates):
|
||||||
|
candidates.append({'id': str(creator.id), 'nick_name': creator.nick_name or creator.username, 'disabled': True})
|
||||||
|
return {
|
||||||
|
'knowledge_id': str(knowledge.id),
|
||||||
|
'user_id': str(knowledge.user_id) if knowledge.user_id else '',
|
||||||
|
'creator_name': (creator.nick_name or creator.username) if creator else '',
|
||||||
|
'create_time': _fmt(knowledge.create_time), 'update_time': _fmt(knowledge.update_time),
|
||||||
|
'creator_options': candidates,
|
||||||
|
'fields': [{'field': {f['label']: f['field'] for f in METADATA_FIELDS}.get(item['name'], item['name']), 'label': item['name'], **item} for item in _metadata_fields(data)],
|
||||||
|
'updated_by': data.get('updated_by_name') or '', 'updated_at': data.get('updated_at') or '',
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
@transaction.atomic
|
||||||
|
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||||||
|
if not isinstance(data, Mapping):
|
||||||
|
raise AppApiException(500, '元数据必须为属性对象')
|
||||||
|
legacy = {item['field']: item['label'] for item in METADATA_FIELDS}
|
||||||
|
if set(data) - (set(legacy) | {'fields', 'user_id'}):
|
||||||
|
raise AppApiException(500, '包含不支持的元数据属性')
|
||||||
|
if 'fields' in data and set(data) & set(legacy):
|
||||||
|
raise AppApiException(500, '不能混用新旧元数据格式')
|
||||||
|
fields = _validate_custom_fields(data['fields']) if 'fields' in data else None
|
||||||
|
values = {}
|
||||||
|
for key, label in legacy.items():
|
||||||
|
if key not in data:
|
||||||
|
continue
|
||||||
|
value = '' if data[key] is None else data[key]
|
||||||
|
if not isinstance(value, str) or len(value.strip()) > METADATA_MAX_LEN:
|
||||||
|
raise AppApiException(500, f'{label}须为64字以内的文本')
|
||||||
|
values[key] = value.strip()
|
||||||
|
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||||||
|
if 'user_id' in data and str(data['user_id']) != str(knowledge.user_id):
|
||||||
|
try:
|
||||||
|
creator_id = serializers.UUIDField().run_validation(data['user_id'])
|
||||||
|
except serializers.ValidationError:
|
||||||
|
raise AppApiException(500, '请选择有效创建者')
|
||||||
|
if not _creator_candidates(workspace_id).filter(id=creator_id).exists():
|
||||||
|
raise AppApiException(500, '创建者须为已激活且启用的成员')
|
||||||
|
knowledge.user_id = creator_id
|
||||||
|
meta = dict(knowledge.meta or {})
|
||||||
|
business = dict(meta.get('business_metadata') or {})
|
||||||
|
if fields is not None:
|
||||||
|
business['custom_fields'] = fields
|
||||||
|
elif 'custom_fields' in business:
|
||||||
|
current = list(_metadata_fields(business))
|
||||||
|
for key, value in values.items():
|
||||||
|
existing = next((item for item in current if item['name'] == legacy[key]), None)
|
||||||
|
if existing is not None:
|
||||||
|
existing['value'] = value
|
||||||
|
elif value:
|
||||||
|
current.append({'name': legacy[key], 'value': value})
|
||||||
|
business['custom_fields'] = _validate_custom_fields(current)
|
||||||
|
business.update(values)
|
||||||
|
business.update(updated_by=str(user.id), updated_by_name=getattr(user, 'nick_name', '') or user.username,
|
||||||
|
updated_at=_fmt(timezone.now()))
|
||||||
|
meta['business_metadata'] = business
|
||||||
|
knowledge.meta = meta
|
||||||
|
knowledge.save(update_fields=['meta', 'user', 'update_time'])
|
||||||
|
return KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeSegmentationSerializer(serializers.Serializer):
|
||||||
|
"""知识库分段规则:查看 / 修改。
|
||||||
|
|
||||||
|
chunk_size 为分段长度,chunk_overlap 为相邻分段重叠长度;
|
||||||
|
保存后新建文档上传时默认使用该规则,上传时可覆盖。
|
||||||
|
"""
|
||||||
|
|
||||||
|
DEFAULTS = {"chunk_size": 500, "chunk_overlap": 0}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def one(workspace_id: str, knowledge_id: str):
|
||||||
|
knowledge = _get_knowledge(workspace_id, knowledge_id)
|
||||||
|
data = dict((knowledge.meta or {}).get('segmentation') or {})
|
||||||
|
result = {
|
||||||
|
"chunk_size": data.get("chunk_size") or KnowledgeSegmentationSerializer.DEFAULTS["chunk_size"],
|
||||||
|
"chunk_overlap": data.get("chunk_overlap") if data.get("chunk_overlap") is not None
|
||||||
|
else KnowledgeSegmentationSerializer.DEFAULTS["chunk_overlap"],
|
||||||
|
}
|
||||||
|
result.update({
|
||||||
|
"updated_by": data.get("updated_by_name") or "",
|
||||||
|
"updated_at": data.get("updated_at") or "",
|
||||||
|
"knowledge_id": str(knowledge.id),
|
||||||
|
})
|
||||||
|
return result
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
@transaction.atomic
|
||||||
|
def update(workspace_id: str, knowledge_id: str, user, data: dict):
|
||||||
|
if not isinstance(data, Mapping):
|
||||||
|
raise AppApiException(500, "分段规则必须为属性对象")
|
||||||
|
def integer(value):
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, str)):
|
||||||
|
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||||||
|
if isinstance(value, str) and not re.fullmatch(r"[+-]?\d+", value):
|
||||||
|
raise AppApiException(500, "分段长度/重叠长度必须为整数")
|
||||||
|
return int(value)
|
||||||
|
chunk_size = integer(data.get("chunk_size"))
|
||||||
|
chunk_overlap = integer(data.get("chunk_overlap", 0))
|
||||||
|
if chunk_size < 50 or chunk_size > 100000:
|
||||||
|
raise AppApiException(500, "分段长度取值范围为 50~100000")
|
||||||
|
if chunk_overlap < 0 or chunk_overlap >= chunk_size:
|
||||||
|
raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度")
|
||||||
|
knowledge = _get_knowledge(workspace_id, knowledge_id, lock=True)
|
||||||
|
meta = dict(knowledge.meta or {})
|
||||||
|
meta['segmentation'] = {
|
||||||
|
"chunk_size": chunk_size,
|
||||||
|
"chunk_overlap": chunk_overlap,
|
||||||
|
"updated_by": str(user.id),
|
||||||
|
"updated_by_name": getattr(user, 'nick_name', '') or user.username,
|
||||||
|
"updated_at": _fmt(timezone.now()),
|
||||||
|
}
|
||||||
|
knowledge.meta = meta
|
||||||
|
knowledge.save(update_fields=["meta", "update_time"])
|
||||||
|
return KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)
|
||||||
|
|
@ -597,7 +597,8 @@ class ParagraphSerializers(serializers.Serializer):
|
||||||
target_knowledge = QuerySet(Knowledge).filter(id=target_knowledge_id).first()
|
target_knowledge = QuerySet(Knowledge).filter(id=target_knowledge_id).first()
|
||||||
knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first()
|
knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first()
|
||||||
embedding_model_id = None
|
embedding_model_id = None
|
||||||
if target_knowledge.embedding_model_id != knowledge.embedding_model_id:
|
if (target_knowledge.embedding_model_id != knowledge.embedding_model_id or
|
||||||
|
(knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})):
|
||||||
embedding_model_id = str(target_knowledge.embedding_model_id)
|
embedding_model_id = str(target_knowledge.embedding_model_id)
|
||||||
pid_list = [paragraph.id for paragraph in paragraph_list]
|
pid_list = [paragraph.id for paragraph in paragraph_list]
|
||||||
# 修改段落信息
|
# 修改段落信息
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,129 @@
|
||||||
|
# coding=utf-8
|
||||||
|
"""知识库独立索引重建任务序列化器。"""
|
||||||
|
from django.db import transaction
|
||||||
|
from django.utils import timezone
|
||||||
|
from rest_framework import serializers
|
||||||
|
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document)
|
||||||
|
from knowledge.task.rebuild import rebuild_knowledge_index
|
||||||
|
from models_provider.models import Model
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt(value):
|
||||||
|
if not value:
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
value = timezone.localtime(value)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return value.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_task(task: KnowledgeRebuildTask):
|
||||||
|
total = task.document_total or 0
|
||||||
|
success = task.document_success or 0
|
||||||
|
failed = task.document_failed or 0
|
||||||
|
processed = success + failed
|
||||||
|
progress = int(processed * 100 / total) if total > 0 else (100 if task.status == RebuildStatus.SUCCESS else 0)
|
||||||
|
status_label = dict(RebuildStatus.choices).get(task.status, task.status)
|
||||||
|
return {
|
||||||
|
"id": str(task.id),
|
||||||
|
"knowledge_id": str(task.knowledge_id),
|
||||||
|
"workspace_id": task.workspace_id,
|
||||||
|
"status": task.status,
|
||||||
|
"status_label": status_label,
|
||||||
|
"config_snapshot": task.config_snapshot or {},
|
||||||
|
"document_total": total,
|
||||||
|
"document_success": success,
|
||||||
|
"document_failed": failed,
|
||||||
|
"progress": progress,
|
||||||
|
"error_message": task.error_message or "",
|
||||||
|
"created_at": _fmt(task.create_time),
|
||||||
|
"started_at": _fmt(task.started_at),
|
||||||
|
"finished_at": _fmt(task.finished_at),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeRebuildSerializer(serializers.Serializer):
|
||||||
|
@staticmethod
|
||||||
|
def create(workspace_id: str, knowledge_id: str, user_id: str):
|
||||||
|
# 锁住知识库行,串行化同一知识库的检查和任务创建。
|
||||||
|
with transaction.atomic():
|
||||||
|
knowledge = Knowledge.objects.select_for_update().filter(
|
||||||
|
id=knowledge_id, workspace_id=workspace_id
|
||||||
|
).first()
|
||||||
|
if knowledge is None:
|
||||||
|
raise AppApiException(500, "知识库不存在")
|
||||||
|
if KnowledgeRebuildTask.objects.filter(
|
||||||
|
knowledge_id=knowledge_id,
|
||||||
|
status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
||||||
|
).exists():
|
||||||
|
raise AppApiException(500, "该知识库已有重建任务在执行")
|
||||||
|
model = Model.objects.filter(id=knowledge.embedding_model_id).first() if knowledge.embedding_model_id else None
|
||||||
|
if model is None:
|
||||||
|
raise AppApiException(500, "知识库未配置可用的向量模型")
|
||||||
|
document_ids = [str(pk) for pk in Document.objects.filter(
|
||||||
|
knowledge_id=knowledge_id).values_list("id", flat=True)]
|
||||||
|
if not document_ids:
|
||||||
|
raise AppApiException(500, "知识库没有可重建的文档")
|
||||||
|
snapshot = {
|
||||||
|
"embedding_model_id": str(model.id),
|
||||||
|
"embedding_params": (knowledge.meta or {}).get("embedding_params", {}),
|
||||||
|
"embedding_model_name": model.name,
|
||||||
|
"embedding_model": model.model_name,
|
||||||
|
"knowledge_name": knowledge.name,
|
||||||
|
"document_ids": document_ids,
|
||||||
|
"document_total": len(document_ids),
|
||||||
|
"snapshot_at": _fmt(timezone.now()),
|
||||||
|
}
|
||||||
|
task = KnowledgeRebuildTask.objects.create(
|
||||||
|
knowledge_id=knowledge_id, workspace_id=workspace_id,
|
||||||
|
status=RebuildStatus.PENDING, config_snapshot=snapshot,
|
||||||
|
document_total=len(document_ids), operator_id=user_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def dispatch():
|
||||||
|
try:
|
||||||
|
result = rebuild_knowledge_index.delay(str(task.id))
|
||||||
|
# 不覆盖 worker 已经更新的状态。
|
||||||
|
KnowledgeRebuildTask.objects.filter(id=task.id).update(
|
||||||
|
celery_task_id=getattr(result, "id", "") or "")
|
||||||
|
except Exception as exc:
|
||||||
|
KnowledgeRebuildTask.objects.filter(id=task.id, status=RebuildStatus.PENDING).update(
|
||||||
|
status=RebuildStatus.FAILED, error_message=f"任务派发失败:{exc}",
|
||||||
|
finished_at=timezone.now(), update_time=timezone.now())
|
||||||
|
|
||||||
|
transaction.on_commit(dispatch)
|
||||||
|
task.refresh_from_db()
|
||||||
|
return serialize_task(task)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def page(workspace_id: str, knowledge_id: str, current_page: int, page_size: int):
|
||||||
|
query_set = KnowledgeRebuildTask.objects.filter(
|
||||||
|
knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||||
|
).order_by("-create_time")
|
||||||
|
total = query_set.count()
|
||||||
|
start = max(0, (current_page - 1) * page_size)
|
||||||
|
records = [serialize_task(item) for item in query_set[start:start + page_size]]
|
||||||
|
return {"records": records, "total": total, "current_page": current_page, "page_size": page_size}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def one(workspace_id: str, knowledge_id: str, task_id: str):
|
||||||
|
task = KnowledgeRebuildTask.objects.filter(
|
||||||
|
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||||
|
).first()
|
||||||
|
if task is None:
|
||||||
|
raise AppApiException(500, "重建任务不存在")
|
||||||
|
return serialize_task(task)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def retry(workspace_id: str, knowledge_id: str, task_id: str, user_id: str):
|
||||||
|
old_task = KnowledgeRebuildTask.objects.filter(
|
||||||
|
id=task_id, knowledge_id=knowledge_id, workspace_id=workspace_id
|
||||||
|
).first()
|
||||||
|
if old_task is None:
|
||||||
|
raise AppApiException(500, "重建任务不存在")
|
||||||
|
if old_task.status not in (RebuildStatus.FAILED, RebuildStatus.PARTIAL):
|
||||||
|
raise AppApiException(500, "仅失败或部分成功的任务可以重新发起")
|
||||||
|
return KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, user_id)
|
||||||
|
|
@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
|
||||||
FROM (SELECT knowledge.*
|
FROM (SELECT knowledge.*
|
||||||
FROM knowledge knowledge ${knowledge_custom_sql}
|
FROM knowledge knowledge ${knowledge_custom_sql}
|
||||||
AND id::text in (select target
|
AND id::text in (select target
|
||||||
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
|
from effective_resource_permission ${workspace_user_resource_permission_query_set}
|
||||||
and 'VIEW' = any (permission_list))) temp_knowledge
|
and 'VIEW' = any (permission_list))) temp_knowledge
|
||||||
LEFT JOIN (SELECT "count"("id") AS document_count, "sum"("char_length") "char_length", knowledge_id
|
LEFT JOIN (SELECT "count"("id") AS document_count, "sum"("char_length") "char_length", knowledge_id
|
||||||
FROM "document"
|
FROM "document"
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
|
||||||
FROM (SELECT knowledge.*
|
FROM (SELECT knowledge.*
|
||||||
FROM knowledge knowledge ${knowledge_custom_sql}
|
FROM knowledge knowledge ${knowledge_custom_sql}
|
||||||
AND "knowledge".id::text in (select target
|
AND "knowledge".id::text in (select target
|
||||||
from workspace_user_resource_permission
|
from effective_resource_permission
|
||||||
${workspace_user_resource_permission_query_set}
|
${workspace_user_resource_permission_query_set}
|
||||||
and case
|
and case
|
||||||
when auth_type = 'ROLE' then
|
when auth_type = 'ROLE' then
|
||||||
|
|
@ -34,8 +34,8 @@ FROM (SELECT "temp_knowledge".id::text, "temp_knowledge".name,
|
||||||
from role_permission role_permission
|
from role_permission role_permission
|
||||||
right join user_role_relation user_role_relation
|
right join user_role_relation user_role_relation
|
||||||
on user_role_relation.role_id=role_permission.role_id
|
on user_role_relation.role_id=role_permission.role_id
|
||||||
where user_role_relation.user_id=workspace_user_resource_permission.user_id
|
where user_role_relation.user_id=effective_resource_permission.user_id
|
||||||
and user_role_relation.workspace_id=workspace_user_resource_permission.workspace_id)
|
and user_role_relation.workspace_id=effective_resource_permission.workspace_id)
|
||||||
else
|
else
|
||||||
'VIEW' = any (permission_list)
|
'VIEW' = any (permission_list)
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from common.config.embedding_config import ModelManage
|
||||||
from common.event.listener_manage import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
|
from common.event.listener_manage import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
|
||||||
UpdateEmbeddingDocumentIdArgs
|
UpdateEmbeddingDocumentIdArgs
|
||||||
from common.utils.logger import maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from knowledge.models import Document, TaskType, State
|
from knowledge.models import Document, Paragraph, Problem, TaskType, State
|
||||||
from knowledge.serializers.common import drop_knowledge_index
|
from knowledge.serializers.common import drop_knowledge_index
|
||||||
from models_provider.models import Model
|
from models_provider.models import Model
|
||||||
from models_provider.tools import get_model, get_model_default_params
|
from models_provider.tools import get_model, get_model_default_params
|
||||||
|
|
@ -35,21 +35,35 @@ def get_embedding_model(model_id, exception_handler=lambda e: maxkb_logger.error
|
||||||
return embedding_model
|
return embedding_model
|
||||||
|
|
||||||
|
|
||||||
|
def scoped_model(model_id, knowledge_id):
|
||||||
|
from knowledge.serializers.common import get_embedding_model_by_knowledge_id
|
||||||
|
if knowledge_id is None:
|
||||||
|
raise ValueError('无法确定向量化任务的知识库')
|
||||||
|
return get_embedding_model_by_knowledge_id(knowledge_id)
|
||||||
|
|
||||||
|
|
||||||
|
def paragraph_knowledge(ids):
|
||||||
|
values = list(QuerySet(Paragraph).filter(id__in=ids).values_list('knowledge_id', flat=True).distinct())
|
||||||
|
if len(values) != 1:
|
||||||
|
raise ValueError('段落须属于同一知识库')
|
||||||
|
return values[0]
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id']}, name='celery:embedding_by_paragraph')
|
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id']}, name='celery:embedding_by_paragraph')
|
||||||
def embedding_by_paragraph(paragraph_id, model_id):
|
def embedding_by_paragraph(paragraph_id, model_id):
|
||||||
embedding_model = get_embedding_model(model_id)
|
embedding_model = scoped_model(model_id, paragraph_knowledge([paragraph_id]))
|
||||||
ListenerManagement.embedding_by_paragraph(paragraph_id, embedding_model)
|
ListenerManagement.embedding_by_paragraph(paragraph_id, embedding_model)
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_data_list')
|
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_data_list')
|
||||||
def embedding_by_paragraph_data_list(data_list, paragraph_id_list, model_id):
|
def embedding_by_paragraph_data_list(data_list, paragraph_id_list, model_id):
|
||||||
embedding_model = get_embedding_model(model_id)
|
embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list))
|
||||||
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model)
|
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model)
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_list')
|
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_list')
|
||||||
def embedding_by_paragraph_list(paragraph_id_list, model_id):
|
def embedding_by_paragraph_list(paragraph_id_list, model_id):
|
||||||
embedding_model = get_embedding_model(model_id)
|
embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list))
|
||||||
ListenerManagement.embedding_by_paragraph_list(paragraph_id_list, embedding_model)
|
ListenerManagement.embedding_by_paragraph_list(paragraph_id_list, embedding_model)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -77,7 +91,11 @@ def embedding_by_document(document_id, model_id, state_list=None):
|
||||||
traceback=traceback.format_exc()
|
traceback=traceback.format_exc()
|
||||||
))
|
))
|
||||||
|
|
||||||
embedding_model = get_embedding_model(model_id, exception_handler)
|
try:
|
||||||
|
embedding_model = scoped_model(model_id, QuerySet(Document).filter(id=document_id).values_list('knowledge_id', flat=True).first())
|
||||||
|
except Exception as exc:
|
||||||
|
exception_handler(exc)
|
||||||
|
raise
|
||||||
#
|
#
|
||||||
ListenerManagement.embedding_by_document(document_id, embedding_model, state_list)
|
ListenerManagement.embedding_by_document(document_id, embedding_model, state_list)
|
||||||
|
|
||||||
|
|
@ -130,12 +148,15 @@ def embedding_by_problem(args, model_id):
|
||||||
@param model_id: 模型id
|
@param model_id: 模型id
|
||||||
@return:
|
@return:
|
||||||
"""
|
"""
|
||||||
embedding_model = get_embedding_model(model_id)
|
embedding_model = scoped_model(model_id, args.get('knowledge_id'))
|
||||||
ListenerManagement.embedding_by_problem(args, embedding_model)
|
ListenerManagement.embedding_by_problem(args, embedding_model)
|
||||||
|
|
||||||
|
|
||||||
def embedding_by_data_list(args: List, model_id):
|
def embedding_by_data_list(args: List, model_id):
|
||||||
embedding_model = get_embedding_model(model_id)
|
if not args:
|
||||||
|
return
|
||||||
|
from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list
|
||||||
|
embedding_model = get_embedding_model_by_knowledge_id_list(list({str(row['knowledge_id']) for row in args}))
|
||||||
ListenerManagement.embedding_by_data_list(args, embedding_model)
|
ListenerManagement.embedding_by_data_list(args, embedding_model)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -220,7 +241,7 @@ def update_problem_embedding(problem_id: str, problem_content: str, model_id):
|
||||||
@param model_id:
|
@param model_id:
|
||||||
@return:
|
@return:
|
||||||
"""
|
"""
|
||||||
model = get_embedding_model(model_id)
|
model = scoped_model(model_id, QuerySet(Problem).filter(id=problem_id).values_list('knowledge_id', flat=True).first())
|
||||||
ListenerManagement.update_problem(UpdateProblemArgs(problem_id, problem_content, model))
|
ListenerManagement.update_problem(UpdateProblemArgs(problem_id, problem_content, model))
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -247,8 +268,8 @@ def delete_embedding_by_paragraph_ids(paragraph_ids: List[str]):
|
||||||
|
|
||||||
def update_embedding_document_id(paragraph_id_list, target_document_id, target_knowledge_id,
|
def update_embedding_document_id(paragraph_id_list, target_document_id, target_knowledge_id,
|
||||||
target_embedding_model_id=None):
|
target_embedding_model_id=None):
|
||||||
target_embedding_model = get_embedding_model(
|
target_embedding_model = scoped_model(
|
||||||
target_embedding_model_id) if target_embedding_model_id is not None else None
|
target_embedding_model_id, target_knowledge_id) if target_embedding_model_id is not None else None
|
||||||
ListenerManagement.update_embedding_document_id(
|
ListenerManagement.update_embedding_document_id(
|
||||||
UpdateEmbeddingDocumentIdArgs(paragraph_id_list, target_document_id, target_knowledge_id,
|
UpdateEmbeddingDocumentIdArgs(paragraph_id_list, target_document_id, target_knowledge_id,
|
||||||
target_embedding_model))
|
target_embedding_model))
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,146 @@
|
||||||
|
# coding=utf-8
|
||||||
|
"""配置不变时独立发起的全量重建;以持久化任务汇总文档结果。"""
|
||||||
|
from django.db import transaction
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from common.db.sql_execute import sql_execute
|
||||||
|
from common.event.listener_manage import ListenerManagement
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
|
from knowledge.models import (Knowledge, KnowledgeRebuildTask, RebuildStatus, Document, Paragraph,
|
||||||
|
TaskType, State, Status)
|
||||||
|
from knowledge.serializers.common import create_knowledge_index, drop_knowledge_index
|
||||||
|
from ops import celery_app
|
||||||
|
|
||||||
|
POLL_SECONDS = 5
|
||||||
|
MAX_POLLS = 120
|
||||||
|
|
||||||
|
|
||||||
|
def _index_exists(knowledge_id: str) -> bool:
|
||||||
|
rows = sql_execute(
|
||||||
|
"SELECT indexname FROM pg_indexes WHERE tablename = 'embedding' AND indexname = %s",
|
||||||
|
[f"embedding_hnsw_idx_{knowledge_id}"]
|
||||||
|
)
|
||||||
|
return bool(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def _fail(task_id, message):
|
||||||
|
KnowledgeRebuildTask.objects.filter(
|
||||||
|
id=task_id, status__in=[RebuildStatus.PENDING, RebuildStatus.RUNNING]
|
||||||
|
).update(status=RebuildStatus.FAILED, error_message=message,
|
||||||
|
finished_at=timezone.now(), update_time=timezone.now())
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name="celery:rebuild_knowledge_index")
|
||||||
|
def rebuild_knowledge_index(task_id: str):
|
||||||
|
# 重复投递不能重复清空向量或重复派发。
|
||||||
|
claimed = KnowledgeRebuildTask.objects.filter(id=task_id, status=RebuildStatus.PENDING).update(
|
||||||
|
status=RebuildStatus.RUNNING, started_at=timezone.now(), error_message="",
|
||||||
|
attempt=0, update_time=timezone.now())
|
||||||
|
if not claimed:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
task = KnowledgeRebuildTask.objects.get(id=task_id)
|
||||||
|
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||||||
|
if knowledge is None:
|
||||||
|
raise ValueError("知识库不存在")
|
||||||
|
snapshot = task.config_snapshot or {}
|
||||||
|
model_id = snapshot.get("embedding_model_id")
|
||||||
|
if not model_id or str(knowledge.embedding_model_id) != model_id:
|
||||||
|
raise ValueError("向量模型已变更,请重新发起任务")
|
||||||
|
if snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {}):
|
||||||
|
raise ValueError('嵌入参数已变更,请重新发起任务')
|
||||||
|
# 兼容历史任务;新任务在受理时即固定文档范围。
|
||||||
|
document_ids = snapshot.get("document_ids")
|
||||||
|
if document_ids is None:
|
||||||
|
document_ids = [str(pk) for pk in Document.objects.filter(
|
||||||
|
knowledge_id=knowledge.id).values_list("id", flat=True)]
|
||||||
|
snapshot["document_ids"] = document_ids
|
||||||
|
task.config_snapshot = snapshot
|
||||||
|
task.save(update_fields=["config_snapshot", "update_time"])
|
||||||
|
documents = list(Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids))
|
||||||
|
if not documents or len(documents) != len(document_ids):
|
||||||
|
raise ValueError("重建范围为空或文档已删除,请重新发起任务")
|
||||||
|
if any(Status(doc.status)[TaskType.EMBEDDING] == State.STARTED for doc in documents):
|
||||||
|
raise ValueError("文档正在向量化,请完成后重新发起")
|
||||||
|
|
||||||
|
ListenerManagement.delete_embedding_by_knowledge(str(knowledge.id))
|
||||||
|
drop_knowledge_index(knowledge_id=str(knowledge.id))
|
||||||
|
task.document_total = len(document_ids)
|
||||||
|
task.save(update_fields=["document_total", "update_time"])
|
||||||
|
# 先重置全部目标文档,避免上一轮 SUCCESS 被本轮汇总误认为已完成。
|
||||||
|
target = Document.objects.filter(knowledge_id=knowledge.id, id__in=document_ids)
|
||||||
|
ListenerManagement.update_status(target, TaskType.EMBEDDING, State.PENDING)
|
||||||
|
ListenerManagement.update_status(
|
||||||
|
Paragraph.objects.filter(knowledge_id=knowledge.id, document_id__in=document_ids),
|
||||||
|
TaskType.EMBEDDING, State.PENDING)
|
||||||
|
from knowledge.task.embedding import embedding_by_document
|
||||||
|
dispatch_errors = []
|
||||||
|
for document in documents:
|
||||||
|
try:
|
||||||
|
embedding_by_document.delay(str(document.id), model_id)
|
||||||
|
except Exception as exc:
|
||||||
|
ListenerManagement.update_status(
|
||||||
|
Document.objects.filter(id=document.id), TaskType.EMBEDDING, State.FAILURE)
|
||||||
|
dispatch_errors.append(f"{document.name}:{exc}")
|
||||||
|
if dispatch_errors:
|
||||||
|
task.error_message = "文档派发失败:" + "; ".join(dispatch_errors)[:2000]
|
||||||
|
task.save(update_fields=["error_message", "update_time"])
|
||||||
|
# 独立轮询不依赖 chord 成功回调;子任务失败也能进入最终汇总。
|
||||||
|
finalize_rebuild_task.apply_async((task_id,), countdown=POLL_SECONDS)
|
||||||
|
except Exception as exc:
|
||||||
|
_fail(task_id, f"重建任务启动失败:{exc}")
|
||||||
|
maxkb_logger.exception("rebuild knowledge index failed: %s", task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(name="celery:finalize_rebuild_task")
|
||||||
|
def finalize_rebuild_task(task_id: str):
|
||||||
|
try:
|
||||||
|
with transaction.atomic():
|
||||||
|
task = KnowledgeRebuildTask.objects.select_for_update().filter(id=task_id).first()
|
||||||
|
if task is None or task.status != RebuildStatus.RUNNING:
|
||||||
|
return
|
||||||
|
snapshot = task.config_snapshot or {}
|
||||||
|
document_ids = snapshot.get("document_ids", [])
|
||||||
|
documents = list(Document.objects.filter(knowledge_id=task.knowledge_id, id__in=document_ids))
|
||||||
|
states = [Status(doc.status)[TaskType.EMBEDDING] for doc in documents]
|
||||||
|
success = states.count(State.SUCCESS)
|
||||||
|
failed = sum(state in (State.FAILURE, State.REVOKE, State.REVOKED, State.IGNORED)
|
||||||
|
for state in states) + len(document_ids) - len(documents)
|
||||||
|
pending = len(document_ids) - success - failed
|
||||||
|
task.document_total = len(document_ids)
|
||||||
|
task.document_success = success
|
||||||
|
task.document_failed = failed
|
||||||
|
if pending:
|
||||||
|
task.attempt += 1
|
||||||
|
if task.attempt < MAX_POLLS:
|
||||||
|
task.save(update_fields=["document_total", "document_success", "document_failed",
|
||||||
|
"attempt", "update_time"])
|
||||||
|
transaction.on_commit(lambda: finalize_rebuild_task.apply_async(
|
||||||
|
(task_id,), countdown=POLL_SECONDS))
|
||||||
|
return
|
||||||
|
task.status = RebuildStatus.FAILED
|
||||||
|
task.error_message = f"重建超时:仍有 {pending} 个文档未完成"
|
||||||
|
else:
|
||||||
|
knowledge_id = str(task.knowledge_id)
|
||||||
|
knowledge = Knowledge.objects.filter(id=task.knowledge_id).first()
|
||||||
|
if (knowledge is None or str(knowledge.embedding_model_id) != snapshot.get("embedding_model_id") or
|
||||||
|
snapshot.get('embedding_params', {}) != (knowledge.meta or {}).get('embedding_params', {})):
|
||||||
|
raise ValueError("向量模型已变更,请重新发起任务")
|
||||||
|
ListenerManagement.get_aggregation_document_status_by_knowledge_id(knowledge_id)()
|
||||||
|
create_knowledge_index(knowledge_id=knowledge_id)
|
||||||
|
index_ready = _index_exists(knowledge_id)
|
||||||
|
if success and failed == 0 and index_ready:
|
||||||
|
task.status = RebuildStatus.SUCCESS
|
||||||
|
task.error_message = ""
|
||||||
|
elif success and failed and index_ready:
|
||||||
|
task.status = RebuildStatus.PARTIAL
|
||||||
|
task.error_message = f"{failed} 个文档失败,{success} 个成功。" + task.error_message
|
||||||
|
else:
|
||||||
|
task.status = RebuildStatus.FAILED
|
||||||
|
task.error_message = f"成功 {success},失败 {failed};索引未通过校验。" + task.error_message
|
||||||
|
task.finished_at = timezone.now()
|
||||||
|
task.save(update_fields=["document_total", "document_success", "document_failed", "status",
|
||||||
|
"error_message", "finished_at", "attempt", "update_time"])
|
||||||
|
except Exception as exc:
|
||||||
|
_fail(task_id, f"重建结果汇总失败:{exc}")
|
||||||
|
maxkb_logger.exception("finalize rebuild task failed: %s", task_id)
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
"""Celery 任务发现入口,确保独立重建和向量化任务被 Worker 注册。"""
|
||||||
|
from knowledge.task import embedding # noqa: F401
|
||||||
|
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task # noqa: F401
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from knowledge.embedding_params import ConfiguredEmbedding, validate_params, knowledge_params
|
||||||
|
|
||||||
|
|
||||||
|
class RecordingModel:
|
||||||
|
batch_size = 10
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def embed_query(self, text):
|
||||||
|
self.calls.append(('query', text))
|
||||||
|
return [3., 4.]
|
||||||
|
|
||||||
|
def embed_documents(self, texts):
|
||||||
|
self.calls.append(('documents', texts))
|
||||||
|
return [[3., 4.] for _ in texts]
|
||||||
|
|
||||||
|
|
||||||
|
class EmbeddingParamsTests(unittest.TestCase):
|
||||||
|
def test_inference_uses_separate_instructions_and_same_normalization(self):
|
||||||
|
base = RecordingModel()
|
||||||
|
model = ConfiguredEmbedding(base, {'normalize': True, 'query_prefix': '检索:', 'document_prefix': '正文:'})
|
||||||
|
self.assertEqual(model.embed_query('内容'), [.6, .8])
|
||||||
|
self.assertEqual(model.embed_documents(['内容']), [[.6, .8]])
|
||||||
|
self.assertEqual(base.calls, [('query', '检索:内容'), ('documents', ['正文:内容'])])
|
||||||
|
|
||||||
|
def test_batch_size_is_applied_to_real_calls(self):
|
||||||
|
base = RecordingModel()
|
||||||
|
model = ConfiguredEmbedding(base, {'batch_size': 2})
|
||||||
|
self.assertEqual(len(model.embed_documents(['a', 'b', 'c', 'd', 'e'])), 5)
|
||||||
|
self.assertEqual([len(call[1]) for call in base.calls], [2, 2, 1])
|
||||||
|
|
||||||
|
def test_knowledge_overrides_do_not_leak_between_shared_models(self):
|
||||||
|
base = RecordingModel()
|
||||||
|
first = ConfiguredEmbedding(base, {'query_prefix': 'A'})
|
||||||
|
second = ConfiguredEmbedding(base, {'query_prefix': 'B'})
|
||||||
|
first.embed_query('x'); second.embed_query('x'); first.embed_query('x')
|
||||||
|
self.assertEqual(base.calls, [('query', 'Ax'), ('query', 'Bx'), ('query', 'Ax')])
|
||||||
|
self.assertEqual(base.batch_size, 10)
|
||||||
|
|
||||||
|
def test_invalid_inputs_are_rejected(self):
|
||||||
|
for params in ([], {'batch_size': True}, {'batch_size': 0}, {'batch_size': 257},
|
||||||
|
{'batch_size': 2.5}, {'normalize': 'true'}, {'query_prefix': 1},
|
||||||
|
{'document_prefix': 'x' * 2001}, {'': 'value'}, {'custom': {}},
|
||||||
|
{'custom': ['value']}, {'x': 1, 'X': 2}, {'custom': float('nan')},
|
||||||
|
{'a' * 65: 'value'}, {str(i): 'value' for i in range(51)}):
|
||||||
|
with self.subTest(params=params), self.assertRaises(ValueError):
|
||||||
|
validate_params(params)
|
||||||
|
|
||||||
|
def test_custom_names_are_saved_but_unsupported_options_do_not_reach_inference(self):
|
||||||
|
params = {'领域': '电力', 'dimensions': 512, '备注': '', 'batch_size': 2, 'query_prefix': '检索:'}
|
||||||
|
self.assertEqual(validate_params(params), params)
|
||||||
|
active = knowledge_params(SimpleNamespace(meta={'embedding_params': params}))
|
||||||
|
self.assertEqual(active, {'batch_size': 2, 'query_prefix': '检索:', 'normalize': True})
|
||||||
|
base = RecordingModel()
|
||||||
|
model = ConfiguredEmbedding(base, active)
|
||||||
|
model.embed_query('问题')
|
||||||
|
self.assertEqual(base.calls, [('query', '检索:问题')])
|
||||||
|
|
||||||
|
def test_missing_normalization_defaults_to_unit_vectors(self):
|
||||||
|
self.assertEqual(knowledge_params(SimpleNamespace(meta={})), {'normalize': True})
|
||||||
|
self.assertEqual(knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': None}})),
|
||||||
|
{'normalize': True})
|
||||||
|
self.assertEqual(validate_params({'batch_size': None, 'query_prefix': ''}), {})
|
||||||
|
base = RecordingModel()
|
||||||
|
self.assertEqual(ConfiguredEmbedding(base, {}).embed_query('x'), [.6, .8])
|
||||||
|
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents(['x']), [[.6, .8]])
|
||||||
|
self.assertEqual(ConfiguredEmbedding(base, {}).embed_documents([]), [])
|
||||||
|
|
||||||
|
def test_explicit_false_keeps_original_vectors(self):
|
||||||
|
params = knowledge_params(SimpleNamespace(meta={'embedding_params': {'normalize': False}}))
|
||||||
|
self.assertEqual(params, {'normalize': False})
|
||||||
|
model = ConfiguredEmbedding(RecordingModel(), params)
|
||||||
|
self.assertEqual(model.embed_query('x'), [3., 4.])
|
||||||
|
self.assertEqual(model.embed_documents(['x']), [[3., 4.]])
|
||||||
|
|
||||||
|
def test_zero_vector_is_safe(self):
|
||||||
|
base = RecordingModel()
|
||||||
|
base.embed_query = lambda _: [0., 0.]
|
||||||
|
self.assertEqual(ConfiguredEmbedding(base, {'normalize': True}).embed_query('x'), [0., 0.])
|
||||||
|
|
@ -0,0 +1,293 @@
|
||||||
|
"""Real isolated persistence and document parsing regression checks."""
|
||||||
|
import io
|
||||||
|
import importlib
|
||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch, Mock
|
||||||
|
from django.test import TestCase
|
||||||
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
from common.utils.split_model import smart_split_paragraph, SplitModel
|
||||||
|
from common.utils.table_split import split_table_rows
|
||||||
|
from knowledge.models import KnowledgeFolder, Knowledge
|
||||||
|
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer as Metadata, KnowledgeSegmentationSerializer as Rules
|
||||||
|
|
||||||
|
|
||||||
|
class MetadataTests(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
KnowledgeFolder.objects.create(id='default', name='root')
|
||||||
|
self.k = Knowledge.objects.create(name='test', desc='', folder_id='default',
|
||||||
|
meta={'internal': {'preserve': True}})
|
||||||
|
self.user = SimpleNamespace(id='test-user', nick_name='测试员', username='tester')
|
||||||
|
|
||||||
|
def test_metadata_persists_and_preserves_other_settings(self):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'business_domain': ' 运维 '})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||||||
|
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||||||
|
result = Metadata.one('default', str(self.k.id))
|
||||||
|
self.assertEqual(len(result['fields']), 1)
|
||||||
|
self.assertEqual(result['updated_by'], '测试员')
|
||||||
|
self.assertTrue(result['updated_at'])
|
||||||
|
|
||||||
|
def test_partial_metadata_update_does_not_clear_other_fields(self):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'owner_dept': '部门'})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['business_metadata']['business_domain'], '运维')
|
||||||
|
|
||||||
|
def test_custom_fields_can_be_created_renamed_deleted_and_cleared(self):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '旧领域'})
|
||||||
|
old = Metadata.one('default', str(self.k.id))
|
||||||
|
self.assertEqual(old['fields'][0]['name'], '业务领域')
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目编号', 'value': ' A-42 '}, {'name': '备注', 'value': ''}]})
|
||||||
|
self.assertEqual([(f['name'], f['value']) for f in Metadata.one('default', str(self.k.id))['fields']], [('项目编号', 'A-42'), ('备注', '')])
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': 'A-43'}]})
|
||||||
|
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['name'], '项目代码')
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'fields': []})
|
||||||
|
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'], [])
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['internal'], {'preserve': True})
|
||||||
|
|
||||||
|
def test_custom_field_validation_does_not_write(self):
|
||||||
|
cases = [[{'name': '', 'value': 'a'}], [{'name': '创建时间', 'value': 'a'}],
|
||||||
|
[{'name': 'Owner', 'value': 'a'}, {'name': 'OWNER', 'value': 'b'}],
|
||||||
|
[{'name': 'a', 'value': 123}], [{'name': 'a', 'value': 'x' * 2001}],
|
||||||
|
[{'name': 'x' * 65, 'value': 'a'}], [{'name': 'a', 'value': '', 'extra': 1}],
|
||||||
|
[{'name': str(i), 'value': ''} for i in range(51)]]
|
||||||
|
for fields in cases:
|
||||||
|
with self.subTest(fields=fields), self.assertRaises(AppApiException):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'fields': fields})
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'create_time': '2020-01-01'})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertNotIn('business_metadata', self.k.meta)
|
||||||
|
|
||||||
|
def test_creator_and_fields_saved_atomically_with_fixed_creation_time(self):
|
||||||
|
from users.models import User
|
||||||
|
creator = User.objects.create(username='creator', nick_name='新创建者', role='USER', is_active=True)
|
||||||
|
original_time = self.k.create_time
|
||||||
|
result = Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.user_id, creator.id)
|
||||||
|
self.assertEqual(self.k.create_time, original_time)
|
||||||
|
self.assertEqual(result['creator_name'], '新创建者')
|
||||||
|
self.assertEqual(result['user_id'], str(creator.id))
|
||||||
|
self.assertTrue(result['update_time'])
|
||||||
|
self.assertIn(str(creator.id), [u['id'] for u in result['creator_options']])
|
||||||
|
|
||||||
|
def test_rejected_pending_disabled_creators_cannot_be_selected(self):
|
||||||
|
from users.models import User, UserActivationApplication
|
||||||
|
creator = User.objects.create(username='pending', nick_name='候选用户', role='USER', is_active=True)
|
||||||
|
application = UserActivationApplication.objects.create(user=creator, status='PENDING')
|
||||||
|
for status in ('PENDING', 'REJECTED'):
|
||||||
|
application.status = status; application.save()
|
||||||
|
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id), 'fields': [{'name': '项目', 'value': '42'}]})
|
||||||
|
self.assertNotIn(str(creator.id), [u['id'] for u in Metadata.one('default', str(self.k.id))['creator_options']])
|
||||||
|
application.status = 'APPROVED'; application.save()
|
||||||
|
creator.is_active = False; creator.save()
|
||||||
|
with self.assertRaises(AppApiException): Metadata.update('default', str(self.k.id), self.user, {'user_id': str(creator.id)})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertIsNone(self.k.user_id)
|
||||||
|
self.assertNotIn('business_metadata', self.k.meta)
|
||||||
|
|
||||||
|
def test_embedding_settings_are_per_knowledge_and_survive_other_saves(self):
|
||||||
|
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||||||
|
other = Knowledge.objects.create(name='other', desc='', folder_id='default', meta={})
|
||||||
|
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||||||
|
operation._validated_data = operation.initial_data
|
||||||
|
operation._errors = {}
|
||||||
|
params = {'batch_size': 3, 'normalize': True, 'query_prefix': '查询:'}
|
||||||
|
with patch.object(operation, 'is_valid'):
|
||||||
|
operation.edit({'embedding_params': params}, select_one=False)
|
||||||
|
operation.edit({'meta': {'selector': 'body', 'embedding_params': {'batch_size': 99}}}, select_one=False)
|
||||||
|
self.k.refresh_from_db(); other.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['embedding_params'], params)
|
||||||
|
self.assertNotIn('embedding_params', other.meta)
|
||||||
|
with patch.object(operation, 'is_valid'), self.assertRaises(AppApiException):
|
||||||
|
operation.edit({'embedding_params': {'batch_size': 0}}, select_one=False)
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['embedding_params'], params)
|
||||||
|
|
||||||
|
def test_general_settings_cannot_restore_stale_metadata(self):
|
||||||
|
from knowledge.serializers.knowledge import KnowledgeSerializer, KnowledgeEditRequest
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'fields': [{'name': '项目代码', 'value': '最新值'}]})
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
protected = self.k.meta['business_metadata']
|
||||||
|
operation = KnowledgeSerializer.Operate(data={'knowledge_id': str(self.k.id), 'workspace_id': 'default', 'user_id': '00000000-0000-4000-8000-000000000001'})
|
||||||
|
operation._validated_data = operation.initial_data
|
||||||
|
operation._errors = {}
|
||||||
|
with patch.object(operation, 'is_valid'), patch.object(KnowledgeEditRequest, 'is_valid'):
|
||||||
|
operation.edit({'meta': {'selector': 'body', 'business_metadata': {'business_domain': '过期值'}, 'parser_id': '伪造服务'}}, select_one=False)
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertEqual(self.k.meta['business_metadata'], protected)
|
||||||
|
self.assertEqual(self.k.meta['selector'], 'body')
|
||||||
|
self.assertNotIn('parser_id', self.k.meta)
|
||||||
|
|
||||||
|
def test_invalid_types_and_unknown_fields_do_not_write(self):
|
||||||
|
for data in ([1], {'business_domain': 123}, {'business_domain': False},
|
||||||
|
{'business_domain': []}, {'business_domain': 'x'*65}, {'internal': 'override'}):
|
||||||
|
with self.subTest(data=data), self.assertRaises(AppApiException):
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, data)
|
||||||
|
self.k.refresh_from_db()
|
||||||
|
self.assertNotIn('business_metadata', self.k.meta)
|
||||||
|
|
||||||
|
def test_workspace_boundary(self):
|
||||||
|
for fn in (Metadata.one, Rules.one):
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
fn('other', str(self.k.id))
|
||||||
|
|
||||||
|
def test_segmentation_and_metadata_updates_preserve_each_other(self):
|
||||||
|
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 100, 'chunk_overlap': 20})
|
||||||
|
Metadata.update('default', str(self.k.id), self.user, {'business_domain': '运维'})
|
||||||
|
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_overlap'], 20)
|
||||||
|
Rules.update('default', str(self.k.id), self.user, {'chunk_size': 200, 'chunk_overlap': 0})
|
||||||
|
self.assertEqual(Metadata.one('default', str(self.k.id))['fields'][0]['value'], '运维')
|
||||||
|
|
||||||
|
def test_segmentation_rejects_invalid_and_fractional_values(self):
|
||||||
|
for size, overlap in ((49,0), (100001,0), (100,100), (100,-1), (100.5,10),
|
||||||
|
(100,1.2), (True,0), (100,False), ('100.5',0)):
|
||||||
|
with self.subTest(size=size, overlap=overlap), self.assertRaises(AppApiException):
|
||||||
|
Rules.update('default', str(self.k.id), self.user, {'chunk_size':size,'chunk_overlap':overlap})
|
||||||
|
|
||||||
|
def test_upload_inherits_defaults_and_accepts_explicit_override(self):
|
||||||
|
from knowledge.serializers.document import DocumentSerializers
|
||||||
|
self.assertEqual(Rules.one('default', str(self.k.id))['chunk_size'], 500)
|
||||||
|
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
|
||||||
|
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
|
||||||
|
serializer.parse({'file':[SimpleUploadedFile('default.txt',b'test')]})
|
||||||
|
self.assertEqual(split.call_args.args[-2:], (500,0))
|
||||||
|
Rules.update('default', str(self.k.id), self.user, {'chunk_size':100,'chunk_overlap':20})
|
||||||
|
serializer = DocumentSerializers.Split(data={'knowledge_id':str(self.k.id),'workspace_id':'default'})
|
||||||
|
with patch.object(serializer,'file_to_paragraph',return_value=[]) as split:
|
||||||
|
f=SimpleUploadedFile('test.txt',b'test')
|
||||||
|
serializer.parse({'file':[f]})
|
||||||
|
self.assertEqual(split.call_args.args[-2:], (100,20))
|
||||||
|
serializer.parse({'file':[f],'limit':'200','overlap':'0'})
|
||||||
|
self.assertEqual(split.call_args.args[-2:], (200,0))
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
serializer.parse({'file':[f],'limit':'100','overlap':'100'})
|
||||||
|
|
||||||
|
|
||||||
|
class ParserTests(unittest.TestCase):
|
||||||
|
text=''.join(chr(0x4e00+i) for i in range(500))
|
||||||
|
|
||||||
|
def check_overlap(self, parts):
|
||||||
|
self.assertGreater(len(parts),1)
|
||||||
|
self.assertTrue(all(len(p)<=100 for p in parts))
|
||||||
|
self.assertTrue(all(parts[i][-20:]==parts[i+1][:20] for i in range(len(parts)-1)))
|
||||||
|
|
||||||
|
def test_plain_text_overlap(self):
|
||||||
|
self.check_overlap(smart_split_paragraph(self.text,100,20))
|
||||||
|
|
||||||
|
def test_pdf_toc_and_internal_link_calls_pass_overlap(self):
|
||||||
|
from common.handle.impl.text.pdf_split_handle import PdfSplitHandle
|
||||||
|
doc=SimpleNamespace(get_toc=lambda:[[1,'chapter',1]],page_count=1,
|
||||||
|
load_page=lambda n:SimpleNamespace(get_text=lambda kind:self.text))
|
||||||
|
result=PdfSplitHandle.handle_toc(doc,100,20)
|
||||||
|
self.check_overlap([p['content'] for p in result])
|
||||||
|
# 两条 PDF 章节路径都必须将 overlap 传至公共分段函数。
|
||||||
|
import ast,inspect,textwrap
|
||||||
|
for method in (PdfSplitHandle.handle_toc, PdfSplitHandle.handle_links):
|
||||||
|
calls=[n for n in ast.walk(ast.parse(textwrap.dedent(inspect.getsource(method))))
|
||||||
|
if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='smart_split_paragraph']
|
||||||
|
self.assertTrue(calls)
|
||||||
|
self.assertTrue(all(len(n.args)==3 for n in calls))
|
||||||
|
|
||||||
|
def test_table_overlap_preserves_rows_headers_and_termination(self):
|
||||||
|
header='| id | value |\n| --- | --- |\n'
|
||||||
|
rows=[f'| {i:02} | example-{i:02} |\n' for i in range(20)]
|
||||||
|
chunks=split_table_rows(header,iter(rows),120,20)
|
||||||
|
self.assertGreater(len(chunks),1)
|
||||||
|
self.assertLess(len(chunks),len(rows))
|
||||||
|
self.assertTrue(all(c['content'].startswith(header) for c in chunks))
|
||||||
|
self.assertTrue(all(len(c['content'])<=120 for c in chunks))
|
||||||
|
for a,b in zip(chunks,chunks[1:]):
|
||||||
|
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||||
|
for row in rows:self.assertTrue(any(row in c['content'] for c in chunks))
|
||||||
|
|
||||||
|
def test_oversized_table_row_is_not_truncated(self):
|
||||||
|
row='| '+'x'*500+' |\n'
|
||||||
|
result=split_table_rows('| id |\n| --- |\n',[row,'| last |\n'],100,20)
|
||||||
|
self.assertTrue(any(row in p['content'] for p in result))
|
||||||
|
self.assertIn('last',result[-1]['content'])
|
||||||
|
|
||||||
|
def test_csv_and_xlsx_use_row_overlap(self):
|
||||||
|
import openpyxl
|
||||||
|
from common.handle.impl.text.csv_split_handle import CsvSplitHandle
|
||||||
|
from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
|
||||||
|
csv=('id,value\n'+''.join(f'{i:02},sample-{i:02}\n' for i in range(20))).encode()
|
||||||
|
book=openpyxl.Workbook();sheet=book.active
|
||||||
|
for row in [('id','value')]+[(str(i).zfill(2),'sample-'+str(i).zfill(2)) for i in range(20)]:sheet.append(row)
|
||||||
|
out=io.BytesIO();book.save(out)
|
||||||
|
for handler,name,data in [(CsvSplitHandle(),'test.csv',csv),(XlsxSplitHandle(),'test.xlsx',out.getvalue())]:
|
||||||
|
f=SimpleUploadedFile(name,data)
|
||||||
|
result=handler.handle(f,[],False,120,lambda _:data,lambda _:None,20)
|
||||||
|
chunks=result[0]['content'] if isinstance(result,list) else result['content']
|
||||||
|
self.assertGreater(len(chunks),1)
|
||||||
|
for a,b in zip(chunks,chunks[1:]):
|
||||||
|
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||||
|
|
||||||
|
def test_xls_uses_row_overlap(self):
|
||||||
|
from common.handle.impl.text.xls_split_handle import handle_sheet
|
||||||
|
data=[['id','value']]+[[f'{i:02}',f'sample-{i:02}'] for i in range(20)]
|
||||||
|
sheet=SimpleNamespace(nrows=len(data),row_values=lambda i:data[i])
|
||||||
|
chunks=handle_sheet('test',sheet,120,20)['content']
|
||||||
|
for a,b in zip(chunks,chunks[1:]):
|
||||||
|
self.assertIn(a['content'].splitlines()[-1],b['content'].splitlines()[2:])
|
||||||
|
|
||||||
|
def test_top_p_migration_preserves_custom_fields_and_is_idempotent(self):
|
||||||
|
migration=importlib.import_module('models_provider.migrations.0002_llm_top_p_params_form')
|
||||||
|
model=SimpleNamespace(model_params_form=[{'field':'custom','default_value':5}],save=Mock())
|
||||||
|
manager=Mock();manager.using.return_value.filter.return_value=[model]
|
||||||
|
apps=SimpleNamespace(get_model=lambda *args:SimpleNamespace(objects=manager))
|
||||||
|
editor=SimpleNamespace(connection=SimpleNamespace(alias='default'))
|
||||||
|
migration.add_top_p_params_form(apps,editor);migration.add_top_p_params_form(apps,editor)
|
||||||
|
self.assertEqual([f['field'] for f in model.model_params_form],['custom','top_p'])
|
||||||
|
model.save.assert_called_once()
|
||||||
|
|
||||||
|
def test_generation_parameters_reach_stream_and_nonstream_http_body(self):
|
||||||
|
import json
|
||||||
|
import httpx
|
||||||
|
from openai import OpenAI
|
||||||
|
from models_provider.impl.aliyun_bai_lian_model_provider.model.llm import BaiLianChatModel
|
||||||
|
from langchain_core.messages import HumanMessage
|
||||||
|
requests=[]
|
||||||
|
def transport(request):
|
||||||
|
body=json.loads(request.content);requests.append(body)
|
||||||
|
if body.get('stream'):
|
||||||
|
events=[]
|
||||||
|
for delta,finish in [({'role':'assistant','content':'ok'},None),({},'stop')]:
|
||||||
|
item={'id':'test','object':'chat.completion.chunk','model':'test','created':1,
|
||||||
|
'choices':[{'index':0,'delta':delta,'finish_reason':finish}]}
|
||||||
|
events.append('data: '+json.dumps(item)+'\n\n')
|
||||||
|
return httpx.Response(200,headers={'Content-Type':'text/event-stream'},
|
||||||
|
text=''.join(events)+'data: [DONE]\n\n')
|
||||||
|
return httpx.Response(200,json={'id':'test','object':'chat.completion','model':'test','created':1,
|
||||||
|
'choices':[{'index':0,'message':{'role':'assistant','content':'ok'},'finish_reason':'stop'}],
|
||||||
|
'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}})
|
||||||
|
with httpx.Client(transport=httpx.MockTransport(transport),trust_env=False) as http:
|
||||||
|
client=OpenAI(api_key='test-only',base_url='http://example.invalid/v1',http_client=http)
|
||||||
|
model=BaiLianChatModel.new_instance('LLM','test',{
|
||||||
|
'api_base':'http://example.invalid/v1','api_key':'test-only'},
|
||||||
|
temperature=0.23,top_p=0.42,max_tokens=10)
|
||||||
|
model.client=client.chat.completions
|
||||||
|
model.root_client=client
|
||||||
|
model.streaming=False
|
||||||
|
self.assertEqual(model.invoke([HumanMessage(content='test')]).content,'ok')
|
||||||
|
model.streaming=True
|
||||||
|
self.assertEqual(''.join(c.content for c in model.stream([HumanMessage(content='test')])),'ok')
|
||||||
|
self.assertEqual(len(requests),2)
|
||||||
|
self.assertEqual([bool(r.get('stream')) for r in requests],[False,True])
|
||||||
|
for req in requests:
|
||||||
|
self.assertEqual(req['temperature'],0.23)
|
||||||
|
self.assertEqual(req['top_p'],0.42)
|
||||||
|
|
||||||
|
def test_new_knowledge_tasks_use_isolated_queue(self):
|
||||||
|
from ops import celery_app
|
||||||
|
import knowledge.tasks
|
||||||
|
for name in ('celery:embedding_by_document','celery:embedding_by_knowledge',
|
||||||
|
'celery:rebuild_knowledge_index','celery:finalize_rebuild_task'):
|
||||||
|
self.assertIn(name,celery_app.tasks)
|
||||||
|
route=celery_app.amqp.router.route({},name)
|
||||||
|
self.assertEqual(route['queue'].name,'uniskb_knowledge')
|
||||||
|
|
@ -0,0 +1,187 @@
|
||||||
|
"""Isolated task persistence and orchestration tests; no broker/model/shared DB calls."""
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch, Mock
|
||||||
|
from contextlib import ExitStack
|
||||||
|
from django.db import transaction, connection
|
||||||
|
from knowledge.models import KnowledgeRebuildTask, RebuildStatus, State
|
||||||
|
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
|
||||||
|
from knowledge.task.rebuild import rebuild_knowledge_index, finalize_rebuild_task, MAX_POLLS
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
|
||||||
|
|
||||||
|
class RebuildTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute('DELETE FROM knowledge_rebuild_task')
|
||||||
|
self.kid = str(uuid.uuid4())
|
||||||
|
self.docid = str(uuid.uuid4())
|
||||||
|
self.modelid = str(uuid.uuid4())
|
||||||
|
self.knowledge = SimpleNamespace(id=self.kid, embedding_model_id=self.modelid, name='test')
|
||||||
|
self.document = SimpleNamespace(id=self.docid, name='test document', status='nn2')
|
||||||
|
self.stack = ExitStack()
|
||||||
|
self.addCleanup(self.stack.close)
|
||||||
|
def mock(name):
|
||||||
|
return self.stack.enter_context(patch(name))
|
||||||
|
self.k = mock('knowledge.serializers.rebuild.Knowledge.objects')
|
||||||
|
self.k.select_for_update.return_value.filter.return_value.first.return_value = self.knowledge
|
||||||
|
self.k.filter.return_value.first.return_value = self.knowledge
|
||||||
|
self.docs = mock('knowledge.serializers.rebuild.Document.objects')
|
||||||
|
self.docs.filter.return_value.values_list.return_value = [self.docid]
|
||||||
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document])
|
||||||
|
self.paragraphs = mock('knowledge.task.rebuild.Paragraph.objects')
|
||||||
|
self.models = mock('knowledge.serializers.rebuild.Model.objects')
|
||||||
|
self.models.filter.return_value.first.return_value = SimpleNamespace(
|
||||||
|
id=self.modelid, name='embedding', model_name='test-model')
|
||||||
|
self.delay = mock('knowledge.serializers.rebuild.rebuild_knowledge_index.delay')
|
||||||
|
self.delay.return_value.id = 'queued-test'
|
||||||
|
self.poll = mock('knowledge.task.rebuild.finalize_rebuild_task.apply_async')
|
||||||
|
self.listener = mock('knowledge.task.rebuild.ListenerManagement')
|
||||||
|
self.drop = mock('knowledge.task.rebuild.drop_knowledge_index')
|
||||||
|
self.create_index = mock('knowledge.task.rebuild.create_knowledge_index')
|
||||||
|
self.index_exists = mock('knowledge.task.rebuild._index_exists')
|
||||||
|
self.index_exists.return_value = True
|
||||||
|
self.child = mock('knowledge.task.embedding.embedding_by_document.delay')
|
||||||
|
|
||||||
|
def create(self):
|
||||||
|
return KnowledgeRebuildSerializer.create('default', self.kid, None)
|
||||||
|
|
||||||
|
def task(self, status=RebuildStatus.RUNNING):
|
||||||
|
return KnowledgeRebuildTask.objects.create(
|
||||||
|
knowledge_id=self.kid, workspace_id='default', status=status, document_total=1,
|
||||||
|
config_snapshot={'embedding_model_id': self.modelid, 'document_ids': [self.docid]})
|
||||||
|
|
||||||
|
def test_dispatch_only_after_commit(self):
|
||||||
|
with transaction.atomic():
|
||||||
|
result = self.create()
|
||||||
|
self.delay.assert_not_called()
|
||||||
|
self.assertEqual(result['status'], RebuildStatus.PENDING)
|
||||||
|
self.delay.assert_called_once()
|
||||||
|
self.assertEqual(KnowledgeRebuildTask.objects.get().celery_task_id, 'queued-test')
|
||||||
|
|
||||||
|
def test_rollback_does_not_dispatch(self):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
with transaction.atomic():
|
||||||
|
self.create()
|
||||||
|
raise ValueError('rollback')
|
||||||
|
self.delay.assert_not_called()
|
||||||
|
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
||||||
|
|
||||||
|
def test_broker_failure_persists_failed_record(self):
|
||||||
|
self.delay.side_effect = RuntimeError('broker offline')
|
||||||
|
result = self.create()
|
||||||
|
self.assertEqual(result['status'], RebuildStatus.FAILED)
|
||||||
|
self.assertIn('broker offline', KnowledgeRebuildTask.objects.get().error_message)
|
||||||
|
|
||||||
|
def test_active_task_rejected_under_knowledge_lock(self):
|
||||||
|
self.task()
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.create()
|
||||||
|
self.k.select_for_update.assert_called_once()
|
||||||
|
self.assertEqual(KnowledgeRebuildTask.objects.count(), 1)
|
||||||
|
|
||||||
|
def test_empty_knowledge_rejected(self):
|
||||||
|
self.docs.filter.return_value.values_list.return_value = []
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.create()
|
||||||
|
self.assertFalse(KnowledgeRebuildTask.objects.exists())
|
||||||
|
|
||||||
|
def test_success_cannot_be_retried(self):
|
||||||
|
task = self.task(RebuildStatus.SUCCESS)
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
KnowledgeRebuildSerializer.retry('default', self.kid, str(task.id), None)
|
||||||
|
|
||||||
|
def test_failed_retry_creates_new_record(self):
|
||||||
|
old = self.task(RebuildStatus.FAILED)
|
||||||
|
result = KnowledgeRebuildSerializer.retry('default', self.kid, str(old.id), None)
|
||||||
|
self.assertNotEqual(result['id'], str(old.id))
|
||||||
|
self.assertEqual(KnowledgeRebuildTask.objects.count(), 2)
|
||||||
|
|
||||||
|
def test_duplicate_worker_delivery_does_not_delete_vectors(self):
|
||||||
|
task = self.task()
|
||||||
|
rebuild_knowledge_index.run(str(task.id))
|
||||||
|
self.listener.delete_embedding_by_knowledge.assert_not_called()
|
||||||
|
|
||||||
|
def test_worker_dispatches_independent_monitor(self):
|
||||||
|
task = self.task(RebuildStatus.PENDING)
|
||||||
|
rebuild_knowledge_index.run(str(task.id))
|
||||||
|
self.child.assert_called_once_with(self.docid, self.modelid)
|
||||||
|
self.paragraphs.filter.assert_called_once_with(knowledge_id=self.kid, document_id__in=[self.docid])
|
||||||
|
self.poll.assert_called_once()
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
||||||
|
|
||||||
|
def test_document_dispatch_failure_is_counted(self):
|
||||||
|
task = self.task(RebuildStatus.PENDING)
|
||||||
|
self.child.side_effect = RuntimeError('cannot queue')
|
||||||
|
rebuild_knowledge_index.run(str(task.id))
|
||||||
|
self.listener.update_status.assert_any_call(
|
||||||
|
self.docs.filter.return_value, unittest.mock.ANY, State.FAILURE)
|
||||||
|
self.document.status = 'nn3'
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
self.assertEqual(task.document_failed, 1)
|
||||||
|
self.assertIn('cannot queue', task.error_message)
|
||||||
|
|
||||||
|
def test_success_requires_index(self):
|
||||||
|
task = self.task()
|
||||||
|
self.index_exists.return_value = False
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
|
||||||
|
def test_successful_finalization(self):
|
||||||
|
task = self.task()
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.SUCCESS)
|
||||||
|
self.assertEqual(task.document_success, 1)
|
||||||
|
self.assertIsNotNone(task.finished_at)
|
||||||
|
|
||||||
|
def test_pending_persists_progress_and_schedules_next_poll(self):
|
||||||
|
task = self.task()
|
||||||
|
other = SimpleNamespace(id=uuid.uuid4(), status='nn0')
|
||||||
|
task.config_snapshot['document_ids'].append(str(other.id))
|
||||||
|
task.save()
|
||||||
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([self.document, other])
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.RUNNING)
|
||||||
|
self.assertEqual(task.document_success, 1)
|
||||||
|
self.poll.assert_called_once()
|
||||||
|
|
||||||
|
def test_timeout_records_failure(self):
|
||||||
|
task = self.task()
|
||||||
|
task.attempt = MAX_POLLS - 1
|
||||||
|
task.save()
|
||||||
|
self.document.status = 'nn0'
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
self.assertIn('超时', task.error_message)
|
||||||
|
|
||||||
|
def test_ignored_document_is_failure(self):
|
||||||
|
task = self.task()
|
||||||
|
self.document.status = 'nnn'
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.document_failed, 1)
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
|
||||||
|
def test_deleted_document_is_not_silent_success(self):
|
||||||
|
task = self.task()
|
||||||
|
self.docs.filter.return_value.__iter__.side_effect = lambda: iter([])
|
||||||
|
finalize_rebuild_task.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.document_failed, 1)
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
|
||||||
|
def test_model_change_fails_before_destructive_work(self):
|
||||||
|
task = self.task(RebuildStatus.PENDING)
|
||||||
|
self.knowledge.embedding_model_id = str(uuid.uuid4())
|
||||||
|
rebuild_knowledge_index.run(str(task.id))
|
||||||
|
task.refresh_from_db()
|
||||||
|
self.assertEqual(task.status, RebuildStatus.FAILED)
|
||||||
|
self.listener.delete_embedding_by_knowledge.assert_not_called()
|
||||||
|
|
@ -0,0 +1,350 @@
|
||||||
|
import uuid
|
||||||
|
import json
|
||||||
|
from datetime import timedelta
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import TestCase
|
||||||
|
from unittest.mock import patch
|
||||||
|
from django.db import transaction
|
||||||
|
from django.test import RequestFactory
|
||||||
|
from django.utils import timezone
|
||||||
|
from rest_framework.exceptions import ValidationError, NotFound
|
||||||
|
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
|
||||||
|
from knowledge import versions as v
|
||||||
|
from knowledge.models import (KnowledgeFolder, Knowledge, Document, Paragraph, Problem, ProblemParagraphMapping,
|
||||||
|
Tag, DocumentTag, Embedding, File, KnowledgeVersion, KnowledgeVersionLease)
|
||||||
|
from users.models import User
|
||||||
|
from models_provider.models import Model
|
||||||
|
from application.models import Application, ApplicationFolder, ApplicationVersion, ApplicationKnowledgeMapping
|
||||||
|
|
||||||
|
|
||||||
|
class VersionTests(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.atomic = transaction.atomic(); self.atomic.__enter__()
|
||||||
|
from django.db import connection
|
||||||
|
if connection.vendor == 'postgresql':
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
|
||||||
|
self.user = User.objects.create(username='CODEX_VERSION_TEST', nick_name='版本测试', role='ADMIN')
|
||||||
|
self.folder = KnowledgeFolder.objects.create(id='default', name='默认')
|
||||||
|
ApplicationFolder.objects.create(id='default', name='默认')
|
||||||
|
self.model = Model.objects.create(name='测试向量', model_name='test-embedding', provider='test',
|
||||||
|
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
|
||||||
|
self.root = Knowledge.objects.create(name='CODEX知识库', desc='原始', folder=self.folder, user=self.user,
|
||||||
|
embedding_model=self.model, meta={'embedding_params': {'batch_size': 8}, 'segmentation': {'chunk_size': 500}})
|
||||||
|
self.doc = Document.objects.create(knowledge=self.root, name='样例', char_length=8, status='nn2')
|
||||||
|
self.para = Paragraph.objects.create(knowledge=self.root, document=self.doc, content='旧答案:100元', position=1)
|
||||||
|
self.problem = Problem.objects.create(knowledge=self.root, content='价格多少')
|
||||||
|
ProblemParagraphMapping.objects.create(knowledge=self.root, document=self.doc, paragraph=self.para, problem=self.problem)
|
||||||
|
tag = Tag.objects.create(knowledge=self.root, key='部门', value='测试')
|
||||||
|
DocumentTag.objects.create(document=self.doc, tag=tag)
|
||||||
|
Embedding.objects.create(id=str(uuid.uuid4()), knowledge=self.root, document=self.doc, paragraph=self.para,
|
||||||
|
source_id=str(self.para.id), source_type='1', embedding='[1,0,0]', search_vector='price')
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
transaction.set_rollback(True); self.atomic.__exit__(None, None, None)
|
||||||
|
|
||||||
|
def create(self, name='初版'):
|
||||||
|
return v.create_version('default', self.root.id, self.user, {'name': name})
|
||||||
|
|
||||||
|
def edit_embedding(self, values):
|
||||||
|
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||||||
|
return KnowledgeSerializer.Operate(data={'user_id': str(self.user.id), 'workspace_id': 'default',
|
||||||
|
'knowledge_id': str(self.root.id)}).edit(values, select_one=False)
|
||||||
|
|
||||||
|
def test_embedding_guard_blocks_version_but_allows_unchanged_and_batch(self):
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
self.create()
|
||||||
|
self.edit_embedding({'embedding_model_id': str(self.model.id), 'embedding_params': {'batch_size': 12}})
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.edit_embedding({'embedding_params': {'batch_size': 12, 'normalize': False}})
|
||||||
|
|
||||||
|
def test_embedding_guard_detects_current_and_published_assistants(self):
|
||||||
|
from common.exception.app_exception import AppApiException
|
||||||
|
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
|
||||||
|
app = Application.objects.create(name='助手', user=self.user)
|
||||||
|
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
|
||||||
|
self.edit_embedding({'embedding_model_id': str(self.model.id)})
|
||||||
|
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=other)
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||||||
|
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||||||
|
'_knowledge_ids': [str(self.root.id), str(other.id)]})
|
||||||
|
ApplicationKnowledgeMapping.objects.filter(application=app, knowledge=other).delete()
|
||||||
|
with self.assertRaises(AppApiException):
|
||||||
|
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||||||
|
ApplicationVersion.objects.create(application=app, knowledge_setting={'_knowledge_ids': [str(self.root.id)]})
|
||||||
|
self.edit_embedding({'embedding_model_id': str(self.model.id)})
|
||||||
|
|
||||||
|
def test_embedding_guard_detects_workflow_and_allows_single_reference(self):
|
||||||
|
from knowledge.embedding_guard import embedding_change_reason, ASSISTANT_REASON
|
||||||
|
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
|
||||||
|
node = {'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||||||
|
'knowledge_id_list': [str(self.root.id)]}}}
|
||||||
|
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user,
|
||||||
|
work_flow={'nodes': [node]})
|
||||||
|
self.assertEqual(embedding_change_reason(self.root), '')
|
||||||
|
node['properties']['node_data']['knowledge_id_list'].append(str(other.id))
|
||||||
|
app.work_flow = {'nodes': [node]}; app.save()
|
||||||
|
self.assertEqual(embedding_change_reason(self.root), ASSISTANT_REASON)
|
||||||
|
|
||||||
|
def test_embedding_guard_allows_model_change_with_single_assistant(self):
|
||||||
|
other_model = Model.objects.create(name='备用向量', model_name='test-embedding-2', provider='test',
|
||||||
|
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
|
||||||
|
app = Application.objects.create(name='助手', user=self.user)
|
||||||
|
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
|
||||||
|
self.edit_embedding({'embedding_model_id': str(other_model.id)})
|
||||||
|
self.root.refresh_from_db()
|
||||||
|
self.assertEqual(self.root.embedding_model_id, other_model.id)
|
||||||
|
|
||||||
|
def test_full_snapshot_and_id_remapping(self):
|
||||||
|
version = self.create()
|
||||||
|
self.assertTrue(version.search_ready)
|
||||||
|
self.assertEqual(version.document_count, 1)
|
||||||
|
paragraph = Paragraph.objects.get(knowledge=version.snapshot)
|
||||||
|
self.assertNotEqual(paragraph.id, self.para.id)
|
||||||
|
self.assertEqual(paragraph.content, self.para.content)
|
||||||
|
mapping = ProblemParagraphMapping.objects.get(knowledge=version.snapshot)
|
||||||
|
self.assertEqual(mapping.paragraph_id, paragraph.id)
|
||||||
|
self.assertEqual(mapping.document_id, paragraph.document_id)
|
||||||
|
self.assertEqual(DocumentTag.objects.filter(document__knowledge=version.snapshot).count(), 1)
|
||||||
|
vector = Embedding.objects.get(knowledge=version.snapshot)
|
||||||
|
self.assertEqual(vector.paragraph_id, paragraph.id)
|
||||||
|
self.assertEqual(vector.source_id, str(paragraph.id))
|
||||||
|
|
||||||
|
def test_live_edits_do_not_change_snapshot(self):
|
||||||
|
version = self.create()
|
||||||
|
self.para.content = '新答案:200元'; self.para.save()
|
||||||
|
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, '旧答案:100元')
|
||||||
|
second = self.create('第二版')
|
||||||
|
self.assertEqual(Paragraph.objects.get(knowledge=second.snapshot).content, '新答案:200元')
|
||||||
|
self.assertNotEqual(version.content_hash, second.content_hash)
|
||||||
|
|
||||||
|
def test_text_uuid_is_preserved(self):
|
||||||
|
self.para.content = '业务标识:' + str(self.doc.id); self.para.save()
|
||||||
|
version = self.create()
|
||||||
|
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, self.para.content)
|
||||||
|
|
||||||
|
def test_binary_files_shared_and_markdown_rewritten(self):
|
||||||
|
file = File(id=uuid.uuid4(), file_name='测试.png', loid=12345, source_id=str(self.root.id),
|
||||||
|
source_type='KNOWLEDGE', sha256_hash='test', file_size=10, meta={'knowledge_id': str(self.root.id)})
|
||||||
|
File.objects.bulk_create([file])
|
||||||
|
self.para.content = ' + ')'; self.para.save()
|
||||||
|
version = self.create()
|
||||||
|
cloned = File.objects.get(source_id=str(version.snapshot_id))
|
||||||
|
self.assertEqual(cloned.loid, file.loid)
|
||||||
|
self.assertNotEqual(cloned.id, file.id)
|
||||||
|
self.assertIn(str(cloned.id), Paragraph.objects.get(knowledge=version.snapshot).content)
|
||||||
|
with patch('knowledge.models.knowledge.select_one') as unlink:
|
||||||
|
v.delete_version('default', self.root.id, version.id)
|
||||||
|
unlink.assert_not_called()
|
||||||
|
self.assertTrue(File.objects.filter(pk=file.id).exists())
|
||||||
|
|
||||||
|
def test_version_delete_cleans_only_snapshot(self):
|
||||||
|
version = self.create(); snapshot_id = version.snapshot_id
|
||||||
|
v.delete_version('default', self.root.id, version.id)
|
||||||
|
for model in (Document, Paragraph, Problem, Embedding, Tag, ProblemParagraphMapping):
|
||||||
|
self.assertFalse(model.objects.filter(knowledge_id=snapshot_id).exists())
|
||||||
|
self.assertFalse(Knowledge.objects.filter(pk=snapshot_id).exists())
|
||||||
|
self.assertTrue(Paragraph.objects.filter(pk=self.para.id).exists())
|
||||||
|
|
||||||
|
def test_version_number_not_reused(self):
|
||||||
|
first = self.create(); v.delete_version('default', self.root.id, first.id)
|
||||||
|
self.assertEqual(self.create().number, 2)
|
||||||
|
|
||||||
|
def test_create_transaction_rolls_back_on_index_failure(self):
|
||||||
|
with patch.object(v, '_build_index', side_effect=RuntimeError('index failed')), self.assertRaises(RuntimeError): self.create()
|
||||||
|
self.assertEqual(Knowledge.objects.count(), 1)
|
||||||
|
self.assertEqual(KnowledgeVersion.objects.count(), 0)
|
||||||
|
|
||||||
|
def test_pending_processing_rejected(self):
|
||||||
|
self.doc.status = 'nn0'; self.doc.save()
|
||||||
|
with self.assertRaises(ValidationError): self.create()
|
||||||
|
|
||||||
|
def test_missing_vectors_are_browsable_but_not_selectable(self):
|
||||||
|
Embedding.objects.all().delete(); version = self.create()
|
||||||
|
self.assertFalse(version.search_ready)
|
||||||
|
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [self.root.id])
|
||||||
|
|
||||||
|
def test_cross_knowledge_selection_rejected(self):
|
||||||
|
version = self.create()
|
||||||
|
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [])
|
||||||
|
|
||||||
|
def test_invalid_version_rejected(self):
|
||||||
|
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): 'invalid'}, 'default', [self.root.id])
|
||||||
|
|
||||||
|
def test_cross_workspace_rejected(self):
|
||||||
|
version = self.create()
|
||||||
|
with self.assertRaises(NotFound404): v.get_version('other', self.root.id, version.id)
|
||||||
|
|
||||||
|
def test_fixed_selection_and_live_lookup(self):
|
||||||
|
version = self.create()
|
||||||
|
self.assertEqual(v.resolve_search([self.root.id], {}, 'default'), [str(self.root.id)])
|
||||||
|
self.assertEqual(v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default'), [str(version.snapshot_id)])
|
||||||
|
|
||||||
|
def test_document_scope_maps_to_frozen_documents(self):
|
||||||
|
version = self.create(); document = Document.objects.get(knowledge=version.snapshot)
|
||||||
|
self.assertEqual(v.resolve_document_scope([self.doc.id], {str(self.root.id): str(version.id)}, 'default'), [str(document.id)])
|
||||||
|
self.assertEqual(v.resolve_document_scope([self.doc.id], {}, 'default'), [str(self.doc.id)])
|
||||||
|
data = v.version_data(version)
|
||||||
|
self.assertNotIn('_document_ids', data['config'])
|
||||||
|
self.assertNotIn('model_identity', data['config'])
|
||||||
|
|
||||||
|
def test_same_chat_keeps_latest_choice(self):
|
||||||
|
first = self.create()
|
||||||
|
selection = {str(self.root.id): 'latest'}
|
||||||
|
pinned = v.pin_selection(selection, 'default', 'chat-test')
|
||||||
|
self.create('新版')
|
||||||
|
self.assertEqual(v.pin_selection(selection, 'default', 'chat-test'), pinned)
|
||||||
|
self.assertEqual(pinned[str(self.root.id)], str(first.id))
|
||||||
|
self.assertNotEqual(v.pin_selection(selection, 'default', 'new-chat'), pinned)
|
||||||
|
|
||||||
|
def test_active_conversation_protects_delete(self):
|
||||||
|
version = self.create(); v.pin_selection({str(self.root.id): str(version.id)}, 'default', 'chat-test')
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||||||
|
KnowledgeVersionLease.objects.update(expires_at=timezone.now()-timedelta(seconds=1))
|
||||||
|
v.delete_version('default', self.root.id, version.id)
|
||||||
|
|
||||||
|
def test_assistant_pin_protects_delete(self):
|
||||||
|
version = self.create()
|
||||||
|
app = Application.objects.create(name='测试助手', user=self.user,
|
||||||
|
knowledge_setting={'version_selection': {str(self.root.id): str(version.id)}})
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||||||
|
app.knowledge_setting = {}; app.save(); v.delete_version('default', self.root.id, version.id)
|
||||||
|
|
||||||
|
def test_latest_assistant_protects_only_latest_ready_version(self):
|
||||||
|
first = self.create(); second = self.create('新版')
|
||||||
|
Application.objects.create(name='测试助手', user=self.user,
|
||||||
|
knowledge_setting={'version_selection': {str(self.root.id): 'latest'}})
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, second.id)
|
||||||
|
v.delete_version('default', self.root.id, first.id)
|
||||||
|
|
||||||
|
def test_workflow_draft_and_published_version_protect_delete(self):
|
||||||
|
version = self.create()
|
||||||
|
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||||||
|
'knowledge_id_list': [str(self.root.id)],
|
||||||
|
'knowledge_setting': {'version_selection': {str(self.root.id): str(version.id)}}}}}]}
|
||||||
|
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, work_flow=flow)
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||||||
|
app.work_flow = {}; app.save()
|
||||||
|
published = ApplicationVersion.objects.create(application=app, type='WORK_FLOW', work_flow=flow)
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||||||
|
published.delete(); v.delete_version('default', self.root.id, version.id)
|
||||||
|
|
||||||
|
def test_workflow_conflicting_versions_rejected(self):
|
||||||
|
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||||||
|
'knowledge_setting': {'version_selection': {str(self.root.id): 'latest'}}}}}]}
|
||||||
|
with self.assertRaises(ValidationError):
|
||||||
|
v.application_selection({'version_selection': {str(self.root.id): 'different'}}, flow)
|
||||||
|
|
||||||
|
def test_republication_releases_superseded_version(self):
|
||||||
|
version = self.create(); app = Application.objects.create(name='重新发布', user=self.user)
|
||||||
|
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||||||
|
'version_selection': {str(self.root.id): str(version.id)}})
|
||||||
|
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||||||
|
ApplicationVersion.objects.create(application=app, knowledge_setting={})
|
||||||
|
v.delete_version('default', self.root.id, version.id)
|
||||||
|
|
||||||
|
def test_migration_target_cannot_be_snapshot(self):
|
||||||
|
version = self.create(); request = RequestFactory().post('/document/migrate')
|
||||||
|
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(self.root.id),
|
||||||
|
'target_knowledge_id': str(version.snapshot_id)})
|
||||||
|
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
|
||||||
|
|
||||||
|
def test_published_chat_keeps_published_association(self):
|
||||||
|
from application.serializers.common import ChatInfo
|
||||||
|
version = self.create()
|
||||||
|
app = Application.objects.create(name='发布测试', user=self.user)
|
||||||
|
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||||||
|
'_knowledge_ids': [str(self.root.id)], 'version_selection': {str(self.root.id): str(version.id)}})
|
||||||
|
self.assertFalse(ApplicationKnowledgeMapping.objects.filter(application=app).exists())
|
||||||
|
chat = ChatInfo(str(uuid.uuid4()), str(self.user.id), 'PLATFORM_USER', [], [], str(app.id))
|
||||||
|
chat.get_application()
|
||||||
|
self.assertEqual(chat.knowledge_id_list, [str(self.root.id)])
|
||||||
|
self.assertEqual(chat.knowledge_version_map, {str(self.root.id): str(version.id)})
|
||||||
|
|
||||||
|
def test_feedback_body_cannot_write_snapshot(self):
|
||||||
|
from application.serializers.application_chat_record import ApplicationChatRecordAddKnowledgeSerializer
|
||||||
|
version = self.create(); doc = Document.objects.get(knowledge=version.snapshot)
|
||||||
|
app = Application.objects.create(name='改进测试', user=self.user)
|
||||||
|
serializer = ApplicationChatRecordAddKnowledgeSerializer(data={'workspace_id': 'default',
|
||||||
|
'application_id': str(app.id), 'knowledge_id': str(version.snapshot_id),
|
||||||
|
'document_id': str(doc.id), 'chat_ids': []})
|
||||||
|
with self.assertRaises(NotFound404): serializer.post_improve({}, with_valid=False)
|
||||||
|
|
||||||
|
def test_model_changes_fail_explicitly(self):
|
||||||
|
version = self.create(); self.model.model_name = 'different'; self.model.save()
|
||||||
|
with self.assertRaises(ValidationError): v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default')
|
||||||
|
|
||||||
|
def test_internal_snapshot_api_guard(self):
|
||||||
|
version = self.create(); request = RequestFactory().get('/admin/api/knowledge')
|
||||||
|
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(version.snapshot_id)})
|
||||||
|
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
|
||||||
|
with self.assertRaises(NotFound404): v.root_knowledge('default', version.snapshot_id)
|
||||||
|
|
||||||
|
def test_api_read_permission_and_workspace(self):
|
||||||
|
from rest_framework.test import APIRequestFactory, force_authenticate
|
||||||
|
from knowledge.views.version import KnowledgeVersionView
|
||||||
|
self.create(); factory = APIRequestFactory()
|
||||||
|
for roles, workspace, expected in [([], 'default', 403), (['ADMIN'], 'default', 200), (['ADMIN'], 'other', 404)]:
|
||||||
|
req = factory.get('/versions')
|
||||||
|
force_authenticate(req, user=self.user, token=SimpleNamespace(role_list=roles, permission_list=[]))
|
||||||
|
response = KnowledgeVersionView.as_view()(req, workspace_id=workspace, knowledge_id=self.root.id)
|
||||||
|
self.assertEqual(response.status_code, expected)
|
||||||
|
|
||||||
|
def test_empty_name_rejected(self):
|
||||||
|
with self.assertRaises(ValidationError): self.create(' ')
|
||||||
|
|
||||||
|
def test_real_postgres_retrieval_reads_frozen_content(self):
|
||||||
|
from django.db import connection
|
||||||
|
if connection.vendor != 'postgresql':
|
||||||
|
self.skipTest('Real pgvector retrieval requires isolated PostgreSQL schema')
|
||||||
|
from application.chat_pipeline.step.search_dataset_step.impl import base_search_dataset_step as search
|
||||||
|
version = self.create()
|
||||||
|
self.para.content = '新答案:200元'; self.para.save()
|
||||||
|
manage = SimpleNamespace(context={'knowledge_version_map': {str(self.root.id): str(version.id)}, 'chat_user_type': 'PLATFORM_USER'})
|
||||||
|
model = SimpleNamespace(model_type='EMBEDDING', name='测试向量')
|
||||||
|
embedding = SimpleNamespace(embed_query=lambda text: [1.0, 0.0, 0.0])
|
||||||
|
with patch.object(search, 'get_model_by_id', return_value=model), patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding):
|
||||||
|
result = search.BaseSearchDatasetStep().execute('价格', [str(self.root.id)], [], [], 3, 0.1,
|
||||||
|
search_mode='embedding', workspace_id='default', manage=manage)
|
||||||
|
self.assertEqual(len(result), 1)
|
||||||
|
self.assertEqual(result[0].content, '旧答案:100元')
|
||||||
|
self.assertIn('v1', result[0].knowledge_name)
|
||||||
|
|
||||||
|
def test_real_postgres_workflow_retrieves_frozen_document(self):
|
||||||
|
from django.db import connection
|
||||||
|
if connection.vendor != 'postgresql': self.skipTest('Real workflow pgvector retrieval requires PostgreSQL')
|
||||||
|
from application.flow.step_node.search_knowledge_node.impl import base_search_knowledge_node as search
|
||||||
|
version = self.create(); self.para.content = '新答案:200元'; self.para.save()
|
||||||
|
body = {'knowledge_version_map': {str(self.root.id): str(version.id)},
|
||||||
|
'chat_user_type': 'PLATFORM_USER', 'workspace_id': 'default'}
|
||||||
|
node = search.BaseSearchKnowledgeNode.__new__(search.BaseSearchKnowledgeNode)
|
||||||
|
node.context = {}; node.workflow_manage = SimpleNamespace(get_body=lambda: body,
|
||||||
|
get_reference_field=lambda *_: [str(self.doc.id)])
|
||||||
|
embedding = SimpleNamespace(embed_query=lambda _: [1.0, 0.0, 0.0])
|
||||||
|
with patch.object(search, 'get_model_instance_by_model_workspace_id'), \
|
||||||
|
patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding), \
|
||||||
|
patch.object(search.connection, 'close'):
|
||||||
|
result = node.execute([str(self.root.id)], {'top_n':3, 'similarity':0.1, 'search_mode':'embedding'},
|
||||||
|
'价格', True, 'referencing', 'document', ['previous', 'documents'])
|
||||||
|
self.assertEqual(result.node_variable['paragraph_list'][0]['content'], '旧答案:100元')
|
||||||
|
|
||||||
|
def test_api_create_browse_and_delete(self):
|
||||||
|
from rest_framework.test import APIRequestFactory, force_authenticate
|
||||||
|
from knowledge.views.version import KnowledgeVersionView
|
||||||
|
factory = APIRequestFactory()
|
||||||
|
def call(method, data=None, version_id=None):
|
||||||
|
request = getattr(factory, method)('/versions', data=data or {}, format='json')
|
||||||
|
force_authenticate(request, user=self.user, token=SimpleNamespace(role_list=['ADMIN'], permission_list=[]))
|
||||||
|
kw = {'workspace_id': 'default', 'knowledge_id': self.root.id}
|
||||||
|
if version_id: kw['version_id'] = version_id
|
||||||
|
return KnowledgeVersionView.as_view()(request, **kw)
|
||||||
|
created = call('post', {'name': '接口创建'})
|
||||||
|
self.assertEqual(json.loads(created.content)['code'], 200)
|
||||||
|
version_id = json.loads(created.content)['data']['id']
|
||||||
|
detail = call('get', version_id=version_id)
|
||||||
|
self.assertEqual(json.loads(detail.content)['data']['documents'][0]['name'], self.doc.name)
|
||||||
|
removed = call('delete', version_id=version_id)
|
||||||
|
self.assertEqual(json.loads(removed.content)['code'], 200)
|
||||||
|
|
@ -1,10 +1,13 @@
|
||||||
from django.urls import path
|
from django.urls import path
|
||||||
|
|
||||||
from . import views
|
from . import views
|
||||||
|
from .views.version import KnowledgeVersionView
|
||||||
|
|
||||||
app_name = "knowledge"
|
app_name = "knowledge"
|
||||||
# @formatter:off
|
# @formatter:off
|
||||||
urlpatterns = [
|
urlpatterns = [
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<uuid:knowledge_id>/versions', KnowledgeVersionView.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<uuid:knowledge_id>/versions/<uuid:version_id>', KnowledgeVersionView.as_view()),
|
||||||
path('workspace/knowledge/document/template/export', views.Template.as_view()),
|
path('workspace/knowledge/document/template/export', views.Template.as_view()),
|
||||||
path('workspace/knowledge/document/table_template/export', views.TableTemplate.as_view()),
|
path('workspace/knowledge/document/table_template/export', views.TableTemplate.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge', views.KnowledgeView.as_view()),
|
path('workspace/<str:workspace_id>/knowledge', views.KnowledgeView.as_view()),
|
||||||
|
|
@ -17,6 +20,12 @@ urlpatterns = [
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/sync', views.KnowledgeView.SyncWeb.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/sync', views.KnowledgeView.SyncWeb.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/generate_related', views.KnowledgeView.GenerateRelated.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/generate_related', views.KnowledgeView.GenerateRelated.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/embedding', views.KnowledgeView.Embedding.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/embedding', views.KnowledgeView.Embedding.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/metadata', views.KnowledgeView.Metadata.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/parser', views.KnowledgeView.Parser.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/segmentation', views.KnowledgeView.Segmentation.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild', views.KnowledgeView.Rebuild.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild/<str:task_id>', views.KnowledgeView.RebuildTask.as_view()),
|
||||||
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/rebuild/<str:task_id>/retry', views.KnowledgeView.RebuildRetry.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/hit_test', views.KnowledgeView.HitTest.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/hit_test', views.KnowledgeView.HitTest.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export', views.KnowledgeView.Export.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export', views.KnowledgeView.Export.as_view()),
|
||||||
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export_zip', views.KnowledgeView.ExportZip.as_view()),
|
path('workspace/<str:workspace_id>/knowledge/<str:knowledge_id>/export_zip', views.KnowledgeView.ExportZip.as_view()),
|
||||||
|
|
|
||||||
|
|
@ -82,7 +82,7 @@ class BaseVectorStore(ABC):
|
||||||
data = {'document_id': document_id, 'paragraph_id': paragraph_id, 'knowledge_id': knowledge_id,
|
data = {'document_id': document_id, 'paragraph_id': paragraph_id, 'knowledge_id': knowledge_id,
|
||||||
'is_active': is_active, 'source_id': source_id, 'source_type': source_type, 'text': text}
|
'is_active': is_active, 'source_id': source_id, 'source_type': source_type, 'text': text}
|
||||||
chunk_list = chunk_data(data)
|
chunk_list = chunk_data(data)
|
||||||
result = sub_array(chunk_list)
|
result = sub_array(chunk_list, self._embedding_batch_size(embedding))
|
||||||
for child_array in result:
|
for child_array in result:
|
||||||
self._batch_save(child_array, embedding, lambda: False)
|
self._batch_save(child_array, embedding, lambda: False)
|
||||||
|
|
||||||
|
|
@ -96,13 +96,22 @@ class BaseVectorStore(ABC):
|
||||||
"""
|
"""
|
||||||
self.save_pre_handler()
|
self.save_pre_handler()
|
||||||
chunk_list = chunk_data_list(data_list)
|
chunk_list = chunk_data_list(data_list)
|
||||||
result = sub_array(chunk_list)
|
result = sub_array(chunk_list, self._embedding_batch_size(embedding))
|
||||||
for child_array in result:
|
for child_array in result:
|
||||||
if not is_the_task_interrupted():
|
if not is_the_task_interrupted():
|
||||||
self._batch_save(child_array, embedding, is_the_task_interrupted)
|
self._batch_save(child_array, embedding, is_the_task_interrupted)
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _embedding_batch_size(embedding):
|
||||||
|
"""向量批次大小:优先取模型参数 batch_size,默认 10。"""
|
||||||
|
try:
|
||||||
|
batch_size = int(getattr(embedding, 'batch_size', 0) or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
batch_size = 0
|
||||||
|
return batch_size if batch_size > 0 else 10
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: str, paragraph_id: str, source_id: str,
|
def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: str, paragraph_id: str, source_id: str,
|
||||||
is_active: bool,
|
is_active: bool,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,331 @@
|
||||||
|
"""Version snapshots reuse existing retrieval storage without exposing another knowledge base."""
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from datetime import timedelta
|
||||||
|
import uuid_utils.compat as uuid
|
||||||
|
from django.db import connection, transaction
|
||||||
|
from django.db.models import Q, OuterRef, Subquery
|
||||||
|
from django.core.exceptions import ValidationError as DjangoValidationError
|
||||||
|
from django.utils import timezone
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
|
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
|
||||||
|
from knowledge.models import (Knowledge, KnowledgeVersion, KnowledgeVersionLease, Document, Paragraph,
|
||||||
|
Problem, ProblemParagraphMapping, Tag, DocumentTag, Embedding, File, TaskType, State, Status)
|
||||||
|
|
||||||
|
SNAPSHOT_KEY = '_version_snapshot'
|
||||||
|
|
||||||
|
|
||||||
|
def root_knowledge(workspace, knowledge_id, lock=False):
|
||||||
|
query = Knowledge.objects.filter(pk=knowledge_id, workspace_id=workspace).exclude(meta__has_key=SNAPSHOT_KEY)
|
||||||
|
if lock:
|
||||||
|
query = query.select_for_update()
|
||||||
|
obj = query.first()
|
||||||
|
if obj is None:
|
||||||
|
raise NotFound404(404, '知识库不存在')
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
def version_data(version):
|
||||||
|
return {'id': str(version.id), 'number': version.number, 'name': version.name,
|
||||||
|
'description': version.description, 'creator_name': version.creator_name,
|
||||||
|
'create_time': version.create_time, 'document_count': version.document_count,
|
||||||
|
'paragraph_count': version.paragraph_count, 'vector_count': version.vector_count,
|
||||||
|
'search_ready': version.search_ready, 'content_hash': version.content_hash,
|
||||||
|
'config': {key: value for key, value in version.config.items()
|
||||||
|
if not key.startswith('_') and key != 'model_identity'}}
|
||||||
|
|
||||||
|
|
||||||
|
def remap(value, mapping):
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k: remap(v, mapping) for k, v in value.items()}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [remap(v, mapping) for v in value]
|
||||||
|
if isinstance(value, str):
|
||||||
|
if value in mapping:
|
||||||
|
return mapping[value]
|
||||||
|
# File links are embedded in Markdown, HTML and document metadata.
|
||||||
|
import re
|
||||||
|
return re.sub(r'(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
|
||||||
|
lambda m: mapping.get(m[0], m[0]), value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def copy_rows(model, source_rows, ids, overrides=None):
|
||||||
|
objects = []
|
||||||
|
for row in source_rows:
|
||||||
|
values = {field.attname: copy.deepcopy(getattr(row, field.attname)) for field in model._meta.concrete_fields}
|
||||||
|
for field in model._meta.concrete_fields:
|
||||||
|
key = field.attname
|
||||||
|
value = values[key]
|
||||||
|
if key.endswith('_id') or key == 'id':
|
||||||
|
values[key] = ids.get(str(value), value)
|
||||||
|
elif key in ('content', 'title', 'name', 'desc') and isinstance(value, str):
|
||||||
|
import re
|
||||||
|
values[key] = re.sub(r'(oss/file/)([0-9a-fA-F-]{36})',
|
||||||
|
lambda match: match[1] + str(ids.get(match[2], match[2])), value)
|
||||||
|
elif isinstance(value, (dict, list, str)):
|
||||||
|
values[key] = remap(value, ids)
|
||||||
|
values.update(overrides or {})
|
||||||
|
objects.append(model(**values))
|
||||||
|
return model.objects.bulk_create(objects, batch_size=500)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_index(snapshot_id):
|
||||||
|
if connection.vendor != 'postgresql':
|
||||||
|
return
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute('SELECT vector_dims(embedding) FROM embedding WHERE knowledge_id=%s LIMIT 1', [str(snapshot_id)])
|
||||||
|
row = cursor.fetchone()
|
||||||
|
if row:
|
||||||
|
# Identifier and predicate are generated UUIDs; dimension comes from pgvector.
|
||||||
|
name = 'embedding_hnsw_idx_' + str(snapshot_id)
|
||||||
|
cursor.execute('CREATE INDEX "' + name + '" ON embedding USING hnsw '
|
||||||
|
'((embedding::vector(' + str(int(row[0])) + ')) vector_cosine_ops) '
|
||||||
|
"WHERE knowledge_id='" + str(snapshot_id) + "'::uuid")
|
||||||
|
|
||||||
|
|
||||||
|
def create_version(workspace, knowledge_id, user, data):
|
||||||
|
name = str(data.get('name') or '').strip()
|
||||||
|
description = str(data.get('description') or '').strip()
|
||||||
|
if not name or len(name) > 128 or len(description) > 512:
|
||||||
|
raise ValidationError('版本名称须为1~128字,说明最多512字')
|
||||||
|
nested = connection.in_atomic_block
|
||||||
|
with transaction.atomic():
|
||||||
|
if connection.vendor == 'postgresql':
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
if nested:
|
||||||
|
cursor.execute('SHOW transaction_isolation')
|
||||||
|
if cursor.fetchone()[0] not in ('repeatable read', 'serializable'):
|
||||||
|
raise ValidationError('版本创建需使用独立的一致性事务')
|
||||||
|
else:
|
||||||
|
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
|
||||||
|
root = root_knowledge(workspace, knowledge_id, lock=True)
|
||||||
|
documents = list(Document.objects.filter(knowledge=root).order_by('id'))
|
||||||
|
if any(Status(doc.status)[kind] in (State.PENDING, State.STARTED)
|
||||||
|
for doc in documents for kind in (TaskType.EMBEDDING, TaskType.GENERATE_PROBLEM, TaskType.SYNC)):
|
||||||
|
raise ValidationError('文档正在处理,请完成后创建版本')
|
||||||
|
paragraphs = list(Paragraph.objects.filter(knowledge=root).order_by('id'))
|
||||||
|
problems = list(Problem.objects.filter(knowledge=root).order_by('id'))
|
||||||
|
mappings = list(ProblemParagraphMapping.objects.filter(knowledge=root))
|
||||||
|
tags = list(Tag.objects.filter(knowledge=root))
|
||||||
|
doc_tags = list(DocumentTag.objects.filter(document__knowledge=root))
|
||||||
|
vectors = list(Embedding.objects.filter(knowledge=root))
|
||||||
|
files = list(File.objects.filter(Q(source_id=str(root.id)) |
|
||||||
|
Q(source_id__in=[str(d.id) for d in documents]) | Q(meta__knowledge_id=str(root.id))))
|
||||||
|
number = root.version_sequence + 1
|
||||||
|
latest = KnowledgeVersion.objects.filter(knowledge=root).first()
|
||||||
|
number = max(number, latest.number + 1 if latest else 1)
|
||||||
|
version_id, snapshot_id = uuid.uuid7(), uuid.uuid7()
|
||||||
|
ids = {str(root.id): str(snapshot_id)}
|
||||||
|
for rows in (documents, paragraphs, problems, mappings, tags, doc_tags, vectors, files):
|
||||||
|
ids.update({str(row.pk): str(uuid.uuid7()) for row in rows})
|
||||||
|
config = {'embedding_model_id': str(root.embedding_model_id) if root.embedding_model_id else None,
|
||||||
|
'embedding_params': (root.meta or {}).get('embedding_params', {}),
|
||||||
|
'segmentation': (root.meta or {}).get('segmentation', {}), 'knowledge_name': root.name,
|
||||||
|
'knowledge_type': root.type, 'metadata': (root.meta or {}).get('business_metadata', {})}
|
||||||
|
model = root.embedding_model
|
||||||
|
if model:
|
||||||
|
config['model_identity'] = {'provider': model.provider, 'model_name': model.model_name,
|
||||||
|
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}
|
||||||
|
content = {'documents': [[str(d.id), d.name, d.is_active, d.meta] for d in documents],
|
||||||
|
'paragraphs': [[str(p.id), str(p.document_id), p.position, p.title, p.content, p.is_active] for p in paragraphs],
|
||||||
|
'problems': [[str(p.id), p.content] for p in problems],
|
||||||
|
'mappings': [[str(p.problem_id), str(p.paragraph_id)] for p in mappings], 'config': config}
|
||||||
|
digest = hashlib.sha256(json.dumps(content, ensure_ascii=False, sort_keys=True, default=str).encode()).hexdigest()
|
||||||
|
config['_document_ids'] = {str(d.id): ids[str(d.id)] for d in documents}
|
||||||
|
metadata = copy.deepcopy(root.meta or {})
|
||||||
|
metadata.pop('_version_sequence', None)
|
||||||
|
metadata.pop('parser_id', None)
|
||||||
|
metadata.update({SNAPSHOT_KEY: str(root.id), '_version_id': str(version_id)})
|
||||||
|
snapshot = Knowledge.objects.create(id=snapshot_id, name=(root.name[:125] + ' · v' + str(number))[:150],
|
||||||
|
desc=root.desc, workspace_id=root.workspace_id, folder_id=root.folder_id, user_id=root.user_id,
|
||||||
|
embedding_model_id=root.embedding_model_id, scope=root.scope, type=0,
|
||||||
|
file_size_limit=root.file_size_limit, file_count_limit=root.file_count_limit, meta=metadata)
|
||||||
|
copy_rows(Document, documents, ids, {'knowledge_id': snapshot_id, 'type': 0})
|
||||||
|
copy_rows(Paragraph, paragraphs, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
|
||||||
|
copy_rows(Problem, problems, ids, {'knowledge_id': snapshot_id, 'hit_num': 0})
|
||||||
|
copy_rows(ProblemParagraphMapping, mappings, ids, {'knowledge_id': snapshot_id})
|
||||||
|
copy_rows(Tag, tags, ids, {'knowledge_id': snapshot_id})
|
||||||
|
copy_rows(DocumentTag, doc_tags, ids)
|
||||||
|
# Preserve binary large-object references; only the last File reference frees the bytes.
|
||||||
|
cloned_files = copy_rows(File, files, ids, {'source_id': str(snapshot_id)})
|
||||||
|
for file in cloned_files:
|
||||||
|
file.meta = {**(file.meta or {}), '_version_id': str(version_id), 'knowledge_id': str(root.id)}
|
||||||
|
File.objects.bulk_update(cloned_files, ['meta'], batch_size=500)
|
||||||
|
copied_vectors = copy_rows(Embedding, vectors, ids, {'knowledge_id': snapshot_id})
|
||||||
|
for old, new in zip(vectors, copied_vectors):
|
||||||
|
new.source_id = ids.get(str(old.source_id), old.source_id)
|
||||||
|
Embedding.objects.bulk_update(copied_vectors, ['source_id'], batch_size=500)
|
||||||
|
active_docs = {d.id for d in documents if d.is_active}
|
||||||
|
active_paragraphs = {p.id for p in paragraphs if p.is_active and p.document_id in active_docs}
|
||||||
|
indexed = {v.paragraph_id for v in vectors if v.is_active}
|
||||||
|
search_ready = bool(model and active_paragraphs and active_paragraphs <= indexed)
|
||||||
|
_build_index(snapshot_id)
|
||||||
|
version = KnowledgeVersion.objects.create(id=version_id, knowledge=root, snapshot=snapshot,
|
||||||
|
number=number, name=name, description=description, creator=user,
|
||||||
|
creator_name=user.nick_name or user.username, document_count=len(documents),
|
||||||
|
paragraph_count=len(paragraphs), vector_count=len(vectors), search_ready=search_ready,
|
||||||
|
content_hash=digest, config=config)
|
||||||
|
root.version_sequence = number
|
||||||
|
root.save(update_fields=['version_sequence', 'update_time'])
|
||||||
|
return version
|
||||||
|
|
||||||
|
|
||||||
|
def get_version(workspace, knowledge_id, version_id, lock=False):
|
||||||
|
root_knowledge(workspace, knowledge_id)
|
||||||
|
query = KnowledgeVersion.objects.filter(knowledge_id=knowledge_id, knowledge__workspace_id=workspace, pk=version_id)
|
||||||
|
if lock:
|
||||||
|
query = query.select_for_update()
|
||||||
|
obj = query.first()
|
||||||
|
if obj is None:
|
||||||
|
raise NotFound404(404, '知识库版本不存在')
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
def validate_selection(selection, workspace, knowledge_ids):
|
||||||
|
if not isinstance(selection, dict) or len(selection) > 100:
|
||||||
|
raise ValidationError('版本选择无效')
|
||||||
|
permitted = {str(k) for k in knowledge_ids}
|
||||||
|
for root_id, chosen in selection.items():
|
||||||
|
if str(root_id) not in permitted:
|
||||||
|
raise ValidationError('只能为已关联的知识库选择版本')
|
||||||
|
if not chosen:
|
||||||
|
continue
|
||||||
|
if chosen == 'latest':
|
||||||
|
version = KnowledgeVersion.objects.filter(knowledge_id=root_id, knowledge__workspace_id=workspace,
|
||||||
|
search_ready=True).first()
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
version = KnowledgeVersion.objects.filter(pk=chosen, knowledge_id=root_id,
|
||||||
|
knowledge__workspace_id=workspace, search_ready=True).first()
|
||||||
|
except (ValueError, DjangoValidationError):
|
||||||
|
version = None
|
||||||
|
if version is None:
|
||||||
|
raise ValidationError('请选择已就绪且属于该知识库的版本')
|
||||||
|
|
||||||
|
|
||||||
|
def application_selection(knowledge_setting, work_flow=None):
|
||||||
|
selection = dict((knowledge_setting or {}).get('version_selection') or {})
|
||||||
|
knowledge_ids = []
|
||||||
|
for node in (work_flow or {}).get('nodes', []):
|
||||||
|
if node.get('type') != 'search-knowledge-node':
|
||||||
|
continue
|
||||||
|
data = node.get('properties', {}).get('node_data', {})
|
||||||
|
knowledge_ids.extend(data.get('knowledge_id_list') or [])
|
||||||
|
for root_id, chosen in (data.get('knowledge_setting') or {}).get('version_selection', {}).items():
|
||||||
|
if root_id in selection and selection[root_id] != chosen:
|
||||||
|
raise ValidationError('同一工作流中的知识库须使用相同版本')
|
||||||
|
selection[root_id] = chosen
|
||||||
|
return selection, knowledge_ids
|
||||||
|
|
||||||
|
|
||||||
|
@transaction.atomic
|
||||||
|
def pin_selection(selection, workspace, chat_id, pinned=None):
|
||||||
|
pinned = dict(pinned or {})
|
||||||
|
for root_id, chosen in sorted((selection or {}).items()):
|
||||||
|
if root_id in pinned or not chosen:
|
||||||
|
continue
|
||||||
|
root_knowledge(workspace, root_id, lock=True)
|
||||||
|
existing = KnowledgeVersionLease.objects.filter(chat_id=str(chat_id), version__knowledge_id=root_id,
|
||||||
|
expires_at__gt=timezone.now()).select_related('version').first()
|
||||||
|
if existing:
|
||||||
|
pinned[str(root_id)] = str(existing.version_id)
|
||||||
|
continue
|
||||||
|
query = KnowledgeVersion.objects.select_for_update().filter(knowledge_id=root_id,
|
||||||
|
knowledge__workspace_id=workspace, search_ready=True)
|
||||||
|
version = query.first() if chosen == 'latest' else query.filter(pk=chosen).first()
|
||||||
|
if version is None:
|
||||||
|
raise ValidationError('选定知识库版本不可用,请检查助手设置')
|
||||||
|
pinned[str(root_id)] = str(version.id)
|
||||||
|
for root_id, version_id in sorted(pinned.items()):
|
||||||
|
root_knowledge(workspace, root_id, lock=True)
|
||||||
|
version = KnowledgeVersion.objects.select_for_update().filter(pk=version_id, knowledge_id=root_id,
|
||||||
|
knowledge__workspace_id=workspace, search_ready=True).first()
|
||||||
|
if version is None:
|
||||||
|
raise ValidationError('会话使用的知识库版本已删除,请新建会话')
|
||||||
|
KnowledgeVersionLease.objects.update_or_create(version=version, chat_id=str(chat_id),
|
||||||
|
defaults={'expires_at': timezone.now() + timedelta(minutes=35)})
|
||||||
|
return pinned
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_search(knowledge_ids, pinned, workspace):
|
||||||
|
result = []
|
||||||
|
for root_id in map(str, knowledge_ids):
|
||||||
|
if root_id not in (pinned or {}):
|
||||||
|
root_knowledge(workspace, root_id)
|
||||||
|
result.append(root_id)
|
||||||
|
continue
|
||||||
|
version = get_version(workspace, root_id, pinned[root_id])
|
||||||
|
if not version.search_ready:
|
||||||
|
raise ValidationError('知识库版本索引不可用')
|
||||||
|
identity = version.config.get('model_identity')
|
||||||
|
model = version.snapshot.embedding_model
|
||||||
|
if not model or (identity and identity != {'provider': model.provider, 'model_name': model.model_name,
|
||||||
|
'credential_hash': hashlib.sha256(model.credential.encode()).hexdigest()}):
|
||||||
|
raise ValidationError('历史版本的向量模型配置已变更,请恢复原模型配置')
|
||||||
|
result.append(str(version.snapshot_id))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_document_scope(document_ids, pinned, workspace):
|
||||||
|
if document_ids is None:
|
||||||
|
return None
|
||||||
|
mapping = {}
|
||||||
|
for root_id, version_id in (pinned or {}).items():
|
||||||
|
mapping.update(get_version(workspace, root_id, version_id).config.get('_document_ids', {}))
|
||||||
|
return [mapping.get(str(document_id), str(document_id)) for document_id in document_ids]
|
||||||
|
|
||||||
|
|
||||||
|
@transaction.atomic
|
||||||
|
def delete_version(workspace, knowledge_id, version_id):
|
||||||
|
root = root_knowledge(workspace, knowledge_id, lock=True)
|
||||||
|
version = get_version(workspace, knowledge_id, version_id, lock=True)
|
||||||
|
from application.models import Application, ApplicationVersion
|
||||||
|
key = 'knowledge_setting__version_selection__' + str(root.id)
|
||||||
|
is_latest = KnowledgeVersion.objects.filter(knowledge=root, search_ready=True).first()
|
||||||
|
# Public chat uses only the latest publication. Superseded application history
|
||||||
|
# must not prevent users from retiring an otherwise unused knowledge version.
|
||||||
|
publications = ApplicationVersion.objects.filter(pk=Subquery(ApplicationVersion.objects.filter(
|
||||||
|
application_id=OuterRef('application_id')).order_by('-create_time', '-id').values('pk')[:1]))
|
||||||
|
for query in (Application.objects.all(), publications):
|
||||||
|
refs = Q(**{key: str(version.id)})
|
||||||
|
if is_latest and is_latest.id == version.id:
|
||||||
|
refs |= Q(**{key: 'latest'})
|
||||||
|
if query.filter(refs).exists():
|
||||||
|
raise ValidationError('版本已被助手或已发布配置使用,请先更换版本并重新发布')
|
||||||
|
for flow in query.filter(type='WORK_FLOW').values_list('work_flow', flat=True).iterator():
|
||||||
|
for node in (flow or {}).get('nodes', []):
|
||||||
|
chosen = ((node.get('properties', {}).get('node_data', {}).get('knowledge_setting') or {})
|
||||||
|
.get('version_selection') or {}).get(str(root.id))
|
||||||
|
if chosen == str(version.id) or (chosen == 'latest' and is_latest and is_latest.id == version.id):
|
||||||
|
raise ValidationError('版本已被工作流或已发布配置使用,请先更换版本并重新发布')
|
||||||
|
KnowledgeVersionLease.objects.filter(version=version, expires_at__lte=timezone.now()).delete()
|
||||||
|
if version.leases.exists():
|
||||||
|
raise ValidationError('版本有活跃会话,请会话结束后再删除')
|
||||||
|
snapshot_id = version.snapshot_id
|
||||||
|
Embedding.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
ProblemParagraphMapping.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
DocumentTag.objects.filter(document__knowledge_id=snapshot_id).delete()
|
||||||
|
Tag.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
Paragraph.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
Document.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
Problem.objects.filter(knowledge_id=snapshot_id).delete()
|
||||||
|
File.objects.filter(source_id=str(snapshot_id)).delete()
|
||||||
|
if connection.vendor == 'postgresql':
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute('DROP INDEX IF EXISTS "embedding_hnsw_idx_' + str(snapshot_id) + '"')
|
||||||
|
version.delete()
|
||||||
|
Knowledge.objects.filter(pk=snapshot_id).delete()
|
||||||
|
|
||||||
|
|
||||||
|
def guard_snapshot_request(request):
|
||||||
|
match = getattr(request, 'resolver_match', None)
|
||||||
|
kwargs = match.kwargs if match else {}
|
||||||
|
knowledge_ids = [kwargs[key] for key in ('knowledge_id', 'target_knowledge_id') if kwargs.get(key)]
|
||||||
|
if knowledge_ids and Knowledge.objects.filter(pk__in=knowledge_ids, meta__has_key=SNAPSHOT_KEY).exists():
|
||||||
|
raise AppUnauthorizedFailed(403, '历史版本只读,请通过原知识库的版本管理查看')
|
||||||
|
file_id = kwargs.get('file_id')
|
||||||
|
if file_id and request.method not in ('GET', 'HEAD', 'OPTIONS') and File.objects.filter(
|
||||||
|
pk=file_id, meta__has_key='_version_id').exists():
|
||||||
|
raise AppUnauthorizedFailed(403, '历史版本文件不能单独删除')
|
||||||
|
|
@ -194,7 +194,7 @@ class DocumentView(APIView):
|
||||||
return result.success(DocumentSerializers.Split(data={
|
return result.success(DocumentSerializers.Split(data={
|
||||||
'workspace_id': workspace_id,
|
'workspace_id': workspace_id,
|
||||||
'knowledge_id': knowledge_id,
|
'knowledge_id': knowledge_id,
|
||||||
}).parse(split_data))
|
}, context={'user': request.user, 'request': request}).parse(split_data))
|
||||||
|
|
||||||
class SplitPattern(APIView):
|
class SplitPattern(APIView):
|
||||||
authentication_classes = [TokenAuth]
|
authentication_classes = [TokenAuth]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from system_manage.ai_hub_knowledge import parser_binding
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
from drf_spectacular.utils import extend_schema
|
from drf_spectacular.utils import extend_schema
|
||||||
from rest_framework.request import Request
|
from rest_framework.request import Request
|
||||||
|
|
@ -14,6 +15,8 @@ from knowledge.api.knowledge import KnowledgeBaseCreateAPI, KnowledgeWebCreateAP
|
||||||
from knowledge.models import KnowledgeScope
|
from knowledge.models import KnowledgeScope
|
||||||
from knowledge.serializers.common import get_knowledge_operation_object
|
from knowledge.serializers.common import get_knowledge_operation_object
|
||||||
from knowledge.serializers.knowledge import KnowledgeSerializer
|
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||||||
|
from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer, KnowledgeSegmentationSerializer
|
||||||
|
from knowledge.serializers.rebuild import KnowledgeRebuildSerializer
|
||||||
from models_provider.serializers.model_serializer import ModelSerializer
|
from models_provider.serializers.model_serializer import ModelSerializer
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -250,6 +253,154 @@ class KnowledgeView(APIView):
|
||||||
data={'knowledge_id': knowledge_id, 'workspace_id': workspace_id, 'user_id': request.user.id}
|
data={'knowledge_id': knowledge_id, 'workspace_id': workspace_id, 'user_id': request.user.id}
|
||||||
).embedding())
|
).embedding())
|
||||||
|
|
||||||
|
class Metadata(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['GET'], summary="知识库业务元数据", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
def get(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(KnowledgeMetadataSerializer.one(workspace_id, knowledge_id))
|
||||||
|
|
||||||
|
@extend_schema(methods=['PUT'], summary="修改知识库业务元数据", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
RoleConstants.ADMIN,
|
||||||
|
)
|
||||||
|
@log(menu='Knowledge Base', operate='Modify knowledge metadata',
|
||||||
|
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
|
||||||
|
def put(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(KnowledgeMetadataSerializer.update(
|
||||||
|
workspace_id, knowledge_id, request.user, request.data))
|
||||||
|
|
||||||
|
class Parser(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['GET'], summary="知识库解析配置", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
def get(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(parser_binding(workspace_id, knowledge_id, request.user))
|
||||||
|
|
||||||
|
@extend_schema(methods=['PUT'], summary="修改知识库解析配置", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
RoleConstants.ADMIN,
|
||||||
|
)
|
||||||
|
@log(menu='Knowledge Base', operate='Modify knowledge parser',
|
||||||
|
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
|
||||||
|
def put(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(parser_binding(workspace_id, knowledge_id, request.user, request.data))
|
||||||
|
|
||||||
|
class Segmentation(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['GET'], summary="知识库分段规则", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
def get(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id))
|
||||||
|
|
||||||
|
@extend_schema(methods=['PUT'], summary="修改知识库分段规则", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
@log(menu='Knowledge Base', operate='Modify knowledge segmentation',
|
||||||
|
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
|
||||||
|
def put(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(KnowledgeSegmentationSerializer.update(
|
||||||
|
workspace_id, knowledge_id, request.user, request.data))
|
||||||
|
|
||||||
|
class Rebuild(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['POST'], summary="独立重建知识库索引", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
@log(menu='Knowledge Base', operate='Rebuild index',
|
||||||
|
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
|
||||||
|
def post(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
return result.success(
|
||||||
|
KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, str(request.user.id))
|
||||||
|
)
|
||||||
|
|
||||||
|
@extend_schema(methods=['GET'], summary="重建任务列表", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
def get(self, request: Request, workspace_id: str, knowledge_id: str):
|
||||||
|
current_page = int(request.query_params.get('current_page', 1))
|
||||||
|
page_size = int(request.query_params.get('page_size', 10))
|
||||||
|
return result.success(
|
||||||
|
KnowledgeRebuildSerializer.page(workspace_id, knowledge_id, current_page, page_size)
|
||||||
|
)
|
||||||
|
|
||||||
|
class RebuildTask(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['GET'], summary="重建任务详情", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
def get(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str):
|
||||||
|
return result.success(
|
||||||
|
KnowledgeRebuildSerializer.one(workspace_id, knowledge_id, task_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
class RebuildRetry(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@extend_schema(methods=['POST'], summary="重试重建任务", tags=["Knowledge Base"])
|
||||||
|
@has_permissions(
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(),
|
||||||
|
PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(),
|
||||||
|
RoleConstants.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([RoleConstants.USER.get_workspace_role()],
|
||||||
|
[PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND),
|
||||||
|
)
|
||||||
|
@log(menu='Knowledge Base', operate='Retry rebuild index',
|
||||||
|
get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id')))
|
||||||
|
def post(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str):
|
||||||
|
return result.success(
|
||||||
|
KnowledgeRebuildSerializer.retry(workspace_id, knowledge_id, task_id, str(request.user.id))
|
||||||
|
)
|
||||||
|
|
||||||
class Export(APIView):
|
class Export(APIView):
|
||||||
authentication_classes = [TokenAuth]
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
from rest_framework.views import APIView
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
|
from common.auth import TokenAuth
|
||||||
|
from common.auth.authentication import has_permissions
|
||||||
|
from common.constants.permission_constants import PermissionConstants as P, RoleConstants as R, ViewPermission, CompareConstants
|
||||||
|
from common.log.log import log
|
||||||
|
from common.result import result
|
||||||
|
from knowledge import versions
|
||||||
|
from knowledge.models import KnowledgeVersion, Document, Paragraph
|
||||||
|
|
||||||
|
|
||||||
|
def rights(action):
|
||||||
|
permission = P.KNOWLEDGE_READ if action == 'read' else P.KNOWLEDGE_EDIT
|
||||||
|
return (R.ADMIN, permission.get_workspace_knowledge_permission(),
|
||||||
|
permission.get_workspace_permission_workspace_manage_role(), R.WORKSPACE_MANAGE.get_workspace_role(),
|
||||||
|
ViewPermission([R.USER.get_workspace_role()], [P.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND))
|
||||||
|
|
||||||
|
|
||||||
|
class KnowledgeVersionView(APIView):
|
||||||
|
authentication_classes = [TokenAuth]
|
||||||
|
|
||||||
|
@has_permissions(*rights('read'))
|
||||||
|
def get(self, request, workspace_id, knowledge_id, version_id=None):
|
||||||
|
versions.root_knowledge(workspace_id, knowledge_id)
|
||||||
|
if not version_id:
|
||||||
|
return result.success([versions.version_data(v) for v in KnowledgeVersion.objects.filter(knowledge_id=knowledge_id)])
|
||||||
|
version = versions.get_version(workspace_id, knowledge_id, version_id)
|
||||||
|
if request.query_params.get('content') == 'paragraphs':
|
||||||
|
try:
|
||||||
|
page = max(1, int(request.query_params.get('page', 1)))
|
||||||
|
size = max(1, min(100, int(request.query_params.get('page_size', 20))))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
raise ValidationError('分页参数无效')
|
||||||
|
query = Paragraph.objects.filter(knowledge_id=version.snapshot_id)
|
||||||
|
document_id = request.query_params.get('document_id')
|
||||||
|
if document_id:
|
||||||
|
query = query.filter(document_id=document_id)
|
||||||
|
search = request.query_params.get('search', '').strip()
|
||||||
|
if search:
|
||||||
|
query = query.filter(content__icontains=search)
|
||||||
|
return result.success({'total': query.count(), 'records': list(query.order_by('document_id', 'position', 'id')
|
||||||
|
.values('id', 'document_id', 'document__name', 'title', 'content', 'position', 'is_active')[(page-1)*size:page*size])})
|
||||||
|
return result.success({**versions.version_data(version), 'documents': list(Document.objects.filter(
|
||||||
|
knowledge_id=version.snapshot_id).order_by('name', 'id').values('id', 'name', 'char_length', 'is_active'))})
|
||||||
|
|
||||||
|
@has_permissions(*rights('edit'))
|
||||||
|
@log(menu='Knowledge Base', operate='Create knowledge version')
|
||||||
|
def post(self, request, workspace_id, knowledge_id):
|
||||||
|
return result.success(versions.version_data(versions.create_version(workspace_id, knowledge_id, request.user, request.data)))
|
||||||
|
|
||||||
|
@has_permissions(*rights('edit'))
|
||||||
|
@log(menu='Knowledge Base', operate='Delete knowledge version')
|
||||||
|
def delete(self, request, workspace_id, knowledge_id, version_id):
|
||||||
|
versions.delete_version(workspace_id, knowledge_id, version_id)
|
||||||
|
return result.success(True)
|
||||||
|
|
@ -4,3 +4,9 @@ from django.apps import AppConfig
|
||||||
class LocalModelConfig(AppConfig):
|
class LocalModelConfig(AppConfig):
|
||||||
default_auto_field = 'django.db.models.BigAutoField'
|
default_auto_field = 'django.db.models.BigAutoField'
|
||||||
name = 'local_model'
|
name = 'local_model'
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
import os
|
||||||
|
if os.environ.get('SERVER_NAME') == 'local_model' and os.environ.get('MAXKB_RUNTIME_AUTOSTART', '1') == '1':
|
||||||
|
from local_model.runtime import start_runtime
|
||||||
|
start_runtime()
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,172 @@
|
||||||
|
"""Weight owner for local resources; shared by both assistants through model IDs."""
|
||||||
|
import gc
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from django.db import close_old_connections
|
||||||
|
|
||||||
|
from common.model_runtime import ModelRuntime, loading_policy
|
||||||
|
|
||||||
|
logger = logging.getLogger('maxkb')
|
||||||
|
SUPPORTED_TYPES = {'local': {'EMBEDDING', 'RERANKER'}, 'ollama': {'LLM', 'EMBEDDING'}}
|
||||||
|
|
||||||
|
|
||||||
|
def driver_for(provider, model_type):
|
||||||
|
driver = {'model_local_provider': 'local', 'model_ollama_provider': 'ollama'}.get(provider)
|
||||||
|
return driver if model_type in SUPPORTED_TYPES.get(driver, set()) else None
|
||||||
|
|
||||||
|
|
||||||
|
def model_spec(model_id):
|
||||||
|
from local_model.models import Model
|
||||||
|
from local_model.serializers.rsa_util import rsa_long_decrypt
|
||||||
|
try:
|
||||||
|
model = Model.objects.get(id=model_id, status='SUCCESS')
|
||||||
|
except Model.DoesNotExist:
|
||||||
|
raise ValueError('模型不存在或尚不可用')
|
||||||
|
driver = driver_for(model.provider, model.model_type)
|
||||||
|
if not driver:
|
||||||
|
raise ValueError('该模型由外部服务运行,不支持本地加载策略')
|
||||||
|
fingerprint = hashlib.sha256((model.provider + model.model_type + model.model_name + model.credential).encode()).hexdigest()
|
||||||
|
return {'id': str(model.id), 'model': model, 'name': model.model_name, 'driver': driver,
|
||||||
|
'type': model.model_type, 'credential': json.loads(rsa_long_decrypt(model.credential)),
|
||||||
|
'fingerprint': fingerprint, 'policy': loading_policy((model.meta or {}).get('loading_policy'))}
|
||||||
|
|
||||||
|
|
||||||
|
class WeightAdapter:
|
||||||
|
@staticmethod
|
||||||
|
def _request(spec, route, data=None):
|
||||||
|
url = urlsplit(spec['credential'].get('api_base', ''))
|
||||||
|
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password:
|
||||||
|
raise ValueError('模型服务地址无效')
|
||||||
|
path = url.path.rstrip('/')
|
||||||
|
if path.endswith('/v1'):
|
||||||
|
path = path[:-3]
|
||||||
|
endpoint = urlunsplit((url.scheme, url.netloc, path + route, '', ''))
|
||||||
|
key = spec['credential'].get('api_key')
|
||||||
|
headers = {'Authorization': 'Bearer ' + key} if key else {}
|
||||||
|
response = requests.request('GET' if data is None else 'POST', endpoint, json=data, headers=headers,
|
||||||
|
timeout=(5, 600 if data is not None else 5), allow_redirects=False)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
def probe(self, spec):
|
||||||
|
try:
|
||||||
|
tags = self._request(spec, '/api/ps').get('models', [])
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
raise ValueError('模型推理服务不可用') from exc
|
||||||
|
name = spec['name'] if ':' in spec['name'].rsplit('/', 1)[-1] else spec['name'] + ':latest'
|
||||||
|
item = next((row for row in tags if row.get('name') == name or row.get('model') == name), None)
|
||||||
|
if item:
|
||||||
|
vram = item.get('size_vram', 0)
|
||||||
|
return {'device': 'GPU' if vram else 'CPU', 'gpu_bytes': vram, 'weight_bytes': item.get('size', 0)}
|
||||||
|
|
||||||
|
def load(self, spec):
|
||||||
|
close_old_connections()
|
||||||
|
try:
|
||||||
|
if spec['driver'] == 'ollama':
|
||||||
|
keep_alive = -1 if spec['policy']['strategy'] == 'PRELOAD' else spec['policy']['idle_seconds']
|
||||||
|
if spec['type'] == 'EMBEDDING':
|
||||||
|
self._request(spec, '/api/embed', {'model': spec['name'], 'input': '模型加载检查', 'keep_alive': keep_alive})
|
||||||
|
else:
|
||||||
|
self._request(spec, '/api/generate', {'model': spec['name'], 'prompt': '', 'stream': False,
|
||||||
|
'keep_alive': keep_alive})
|
||||||
|
info = self.probe(spec)
|
||||||
|
if not info:
|
||||||
|
raise ValueError('模型未驻留')
|
||||||
|
logger.info('Model weights ready: %s', spec['id'])
|
||||||
|
return True, info
|
||||||
|
from local_model.serializers.model_apply_serializers import get_local_model
|
||||||
|
instance = get_local_model(spec['model'], use_local=True)
|
||||||
|
if spec['type'] == 'EMBEDDING':
|
||||||
|
instance.embed_query('模型加载检查')
|
||||||
|
else:
|
||||||
|
from langchain_core.documents import Document
|
||||||
|
instance.compress_documents([Document(page_content='模型加载检查')], '模型')
|
||||||
|
client = getattr(instance, '_client', None) or getattr(instance, 'client', None)
|
||||||
|
device = str(getattr(client, 'device', spec['credential'].get('device') or 'cpu'))
|
||||||
|
info = {'device': device}
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
if device.startswith('cuda'):
|
||||||
|
info['process_gpu_bytes'] = torch.cuda.memory_allocated(device)
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
logger.info('Model weights ready: %s', spec['id'])
|
||||||
|
return instance, info
|
||||||
|
finally:
|
||||||
|
close_old_connections()
|
||||||
|
|
||||||
|
def unload(self, spec, value):
|
||||||
|
if spec['driver'] == 'ollama':
|
||||||
|
# Refuse to unload a native model shared by another configured resource.
|
||||||
|
from local_model.models import Model
|
||||||
|
if Model.objects.filter(provider='model_ollama_provider', model_name=spec['name']).exclude(id=spec['id']).exists():
|
||||||
|
raise ValueError('该模型被其他模型资源共用,不能单独卸载')
|
||||||
|
if spec['type'] == 'EMBEDDING':
|
||||||
|
self._request(spec, '/api/embed', {'model': spec['name'], 'input': [], 'keep_alive': 0})
|
||||||
|
else:
|
||||||
|
self._request(spec, '/api/generate', {'model': spec['name'], 'keep_alive': 0, 'stream': False})
|
||||||
|
for _ in range(50):
|
||||||
|
if self.probe(spec) is None:
|
||||||
|
return
|
||||||
|
time.sleep(.2)
|
||||||
|
raise ValueError('模型卸载未完成,请稍后重试')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def collect():
|
||||||
|
gc.collect()
|
||||||
|
try:
|
||||||
|
import torch
|
||||||
|
if torch.cuda.is_initialized():
|
||||||
|
torch.cuda.empty_cache()
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def error(exc):
|
||||||
|
name = type(exc).__name__
|
||||||
|
logger.warning('Model weight load failed: %s', name)
|
||||||
|
if 'OutOfMemory' in name:
|
||||||
|
return '模型加载失败:显存或内存不足'
|
||||||
|
return '模型加载失败,请检查模型文件、运行设备及服务状态'
|
||||||
|
|
||||||
|
|
||||||
|
manager = ModelRuntime(model_spec, WeightAdapter())
|
||||||
|
_startup_lock = threading.Lock()
|
||||||
|
_started = False
|
||||||
|
|
||||||
|
|
||||||
|
def start_runtime():
|
||||||
|
global _started
|
||||||
|
with _startup_lock:
|
||||||
|
if _started:
|
||||||
|
return
|
||||||
|
_started = True
|
||||||
|
def run():
|
||||||
|
from django.apps import apps
|
||||||
|
while not apps.ready:
|
||||||
|
time.sleep(.05)
|
||||||
|
from local_model.models import Model
|
||||||
|
try:
|
||||||
|
models = Model.objects.filter(status='SUCCESS', meta__loading_policy__strategy='PRELOAD')
|
||||||
|
ids = [str(m.id) for m in models if driver_for(m.provider, m.model_type)]
|
||||||
|
manager.preload(ids)
|
||||||
|
logger.info('Startup model preloads queued: %s', len(ids))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning('Startup preload discovery failed: %s', type(exc).__name__)
|
||||||
|
finally:
|
||||||
|
close_old_connections()
|
||||||
|
while True:
|
||||||
|
time.sleep(10)
|
||||||
|
try:
|
||||||
|
manager.sweep()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning('Model idle sweep failed: %s', type(exc).__name__)
|
||||||
|
finally:
|
||||||
|
close_old_connections()
|
||||||
|
threading.Thread(target=run, name='model-startup', daemon=True).start()
|
||||||
|
|
@ -7,72 +7,14 @@
|
||||||
@desc:
|
@desc:
|
||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
|
|
||||||
from django.db import connection
|
|
||||||
from django.db.models import QuerySet
|
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
from langchain_core.documents import Document
|
from langchain_core.documents import Document
|
||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
from local_model.models import Model
|
|
||||||
from local_model.serializers.rsa_util import rsa_long_decrypt
|
from local_model.serializers.rsa_util import rsa_long_decrypt
|
||||||
from models_provider.impl.local_model_provider.local_model_provider import LocalModelProvider
|
from models_provider.impl.local_model_provider.local_model_provider import LocalModelProvider
|
||||||
|
|
||||||
from common.cache.mem_cache import MemCache
|
|
||||||
|
|
||||||
_lock = threading.Lock()
|
|
||||||
locks = {}
|
|
||||||
|
|
||||||
|
|
||||||
class ModelManage:
|
|
||||||
cache = MemCache('model', {})
|
|
||||||
up_clear_time = time.time()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _get_lock(_id):
|
|
||||||
lock = locks.get(_id)
|
|
||||||
if lock is None:
|
|
||||||
with _lock:
|
|
||||||
lock = locks.get(_id)
|
|
||||||
if lock is None:
|
|
||||||
lock = threading.Lock()
|
|
||||||
locks[_id] = lock
|
|
||||||
|
|
||||||
return lock
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_model(_id, get_model):
|
|
||||||
model_instance = ModelManage.cache.get(_id)
|
|
||||||
if model_instance is None:
|
|
||||||
lock = ModelManage._get_lock(_id)
|
|
||||||
with lock:
|
|
||||||
model_instance = ModelManage.cache.get(_id)
|
|
||||||
if model_instance is None:
|
|
||||||
model_instance = get_model(_id)
|
|
||||||
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
|
|
||||||
else:
|
|
||||||
if model_instance.is_cache_model():
|
|
||||||
ModelManage.cache.touch(_id, timeout=60 * 60 * 8)
|
|
||||||
else:
|
|
||||||
model_instance = get_model(_id)
|
|
||||||
ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
|
|
||||||
ModelManage.clear_timeout_cache()
|
|
||||||
return model_instance
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def clear_timeout_cache():
|
|
||||||
if time.time() - ModelManage.up_clear_time > 60 * 60:
|
|
||||||
threading.Thread(target=lambda: ModelManage.cache.clear_timeout_data()).start()
|
|
||||||
ModelManage.up_clear_time = time.time()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def delete_key(_id):
|
|
||||||
if ModelManage.cache.has_key(_id):
|
|
||||||
ModelManage.cache.delete(_id)
|
|
||||||
|
|
||||||
|
|
||||||
def get_local_model(model, **kwargs):
|
def get_local_model(model, **kwargs):
|
||||||
return LocalModelProvider().get_model(model.model_type, model.model_name,
|
return LocalModelProvider().get_model(model.model_type, model.model_name,
|
||||||
json.loads(
|
json.loads(
|
||||||
|
|
@ -81,19 +23,10 @@ def get_local_model(model, **kwargs):
|
||||||
streaming=True, **kwargs)
|
streaming=True, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model(model_id):
|
|
||||||
model = QuerySet(Model).filter(id=model_id).first()
|
|
||||||
# 手动关闭数据库连接
|
|
||||||
connection.close()
|
|
||||||
embedding_model = ModelManage.get_model(model_id,
|
|
||||||
lambda _id: get_local_model(model, use_local=True))
|
|
||||||
return embedding_model
|
|
||||||
|
|
||||||
|
|
||||||
class EmbedDocuments(serializers.Serializer):
|
class EmbedDocuments(serializers.Serializer):
|
||||||
texts = serializers.ListField(required=True, child=serializers.CharField(required=True,
|
texts = serializers.ListField(required=True, child=serializers.CharField(required=True,
|
||||||
label=_('vector text')),
|
label=_('vector text')),
|
||||||
label=_('vector text list')),
|
label=_('vector text list'))
|
||||||
|
|
||||||
|
|
||||||
class EmbedQuery(serializers.Serializer):
|
class EmbedQuery(serializers.Serializer):
|
||||||
|
|
@ -133,22 +66,26 @@ class ModelApplySerializers(serializers.Serializer):
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
EmbedDocuments(data=instance).is_valid(raise_exception=True)
|
EmbedDocuments(data=instance).is_valid(raise_exception=True)
|
||||||
|
|
||||||
model = get_embedding_model(self.data.get('model_id'))
|
from local_model.runtime import manager
|
||||||
return model.embed_documents(instance.getlist('texts'))
|
with manager.inference(self.data.get('model_id')) as model:
|
||||||
|
texts = instance.getlist('texts') if hasattr(instance, 'getlist') else instance.get('texts')
|
||||||
|
return model.embed_documents(texts)
|
||||||
|
|
||||||
def embed_query(self, instance, with_valid=True):
|
def embed_query(self, instance, with_valid=True):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
EmbedQuery(data=instance).is_valid(raise_exception=True)
|
EmbedQuery(data=instance).is_valid(raise_exception=True)
|
||||||
|
|
||||||
model = get_embedding_model(self.data.get('model_id'))
|
from local_model.runtime import manager
|
||||||
|
with manager.inference(self.data.get('model_id')) as model:
|
||||||
return model.embed_query(instance.get('text'))
|
return model.embed_query(instance.get('text'))
|
||||||
|
|
||||||
def compress_documents(self, instance, with_valid=True):
|
def compress_documents(self, instance, with_valid=True):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
CompressDocuments(data=instance).is_valid(raise_exception=True)
|
CompressDocuments(data=instance).is_valid(raise_exception=True)
|
||||||
model = get_embedding_model(self.data.get('model_id'))
|
from local_model.runtime import manager
|
||||||
|
with manager.inference(self.data.get('model_id')) as model:
|
||||||
return [{'page_content': d.page_content, 'metadata': d.metadata} for d in model.compress_documents(
|
return [{'page_content': d.page_content, 'metadata': d.metadata} for d in model.compress_documents(
|
||||||
[Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document in
|
[Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document in
|
||||||
instance.get('documents')], instance.get('query'))]
|
instance.get('documents')], instance.get('query'))]
|
||||||
|
|
@ -156,5 +93,6 @@ class ModelApplySerializers(serializers.Serializer):
|
||||||
def unload(self, with_valid=True):
|
def unload(self, with_valid=True):
|
||||||
if with_valid:
|
if with_valid:
|
||||||
self.is_valid(raise_exception=True)
|
self.is_valid(raise_exception=True)
|
||||||
ModelManage.delete_key(self.data.get('model_id'))
|
from local_model.runtime import manager
|
||||||
|
manager.unload(self.data.get('model_id'))
|
||||||
return True
|
return True
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,12 @@ import os
|
||||||
from django.urls import path
|
from django.urls import path
|
||||||
|
|
||||||
from . import views
|
from . import views
|
||||||
|
from .views.runtime import LocalModelRuntime
|
||||||
|
|
||||||
app_name = "local_model"
|
app_name = "local_model"
|
||||||
# @formatter:off
|
# @formatter:off
|
||||||
urlpatterns = [
|
urlpatterns = [
|
||||||
|
path('model_runtime', LocalModelRuntime.as_view()),
|
||||||
path('model/validate', views.LocalModelApply.Validate.as_view()),
|
path('model/validate', views.LocalModelApply.Validate.as_view()),
|
||||||
path('model/<str:model_id>/embed_documents', views.LocalModelApply.EmbedDocuments.as_view()),
|
path('model/<str:model_id>/embed_documents', views.LocalModelApply.EmbedDocuments.as_view()),
|
||||||
path('model/<str:model_id>/embed_query', views.LocalModelApply.EmbedQuery.as_view()),
|
path('model/<str:model_id>/embed_query', views.LocalModelApply.EmbedQuery.as_view()),
|
||||||
|
|
|
||||||
|
|
@ -35,8 +35,9 @@ class LocalModelApply(APIView):
|
||||||
|
|
||||||
class Unload(APIView):
|
class Unload(APIView):
|
||||||
def post(self, request: Request, model_id):
|
def post(self, request: Request, model_id):
|
||||||
return result.success(
|
# Administrative weight operations use the signed runtime endpoint.
|
||||||
ModelApplySerializers(data={'model_id': model_id}).compress_documents(request.data))
|
from rest_framework.exceptions import PermissionDenied
|
||||||
|
raise PermissionDenied('请通过模型管理卸载')
|
||||||
|
|
||||||
class Validate(APIView):
|
class Validate(APIView):
|
||||||
def post(self, request: Request):
|
def post(self, request: Request):
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
import re
|
||||||
|
|
||||||
|
from rest_framework.exceptions import PermissionDenied, ValidationError
|
||||||
|
from rest_framework.serializers import UUIDField
|
||||||
|
from rest_framework.views import APIView
|
||||||
|
|
||||||
|
from common.model_runtime_client import verify_request
|
||||||
|
from common.auth.authenticate import AnonymousAuthentication
|
||||||
|
from common.result import result
|
||||||
|
from local_model.runtime import manager
|
||||||
|
|
||||||
|
|
||||||
|
class LocalModelRuntime(APIView):
|
||||||
|
authentication_classes = [AnonymousAuthentication]
|
||||||
|
permission_classes = []
|
||||||
|
|
||||||
|
def post(self, request):
|
||||||
|
try:
|
||||||
|
data = verify_request(request.body, request.headers.get('X-Model-Runtime-Signature'))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
raise PermissionDenied('模型服务认证失败')
|
||||||
|
model_id = str(UUIDField().run_validation(data.get('model_id')))
|
||||||
|
operation = data.get('operation')
|
||||||
|
try:
|
||||||
|
if operation == 'release':
|
||||||
|
lease = data.get('lease', '')
|
||||||
|
if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{32}', lease):
|
||||||
|
raise ValueError('请求标识无效')
|
||||||
|
manager.release(model_id, lease)
|
||||||
|
return result.success({'released': True})
|
||||||
|
if operation == 'acquire':
|
||||||
|
lease = manager.acquire(model_id, remote=True)
|
||||||
|
try:
|
||||||
|
return result.success({**manager.status(model_id), 'lease': lease})
|
||||||
|
except Exception:
|
||||||
|
manager.release(model_id, lease)
|
||||||
|
raise
|
||||||
|
if operation == 'load':
|
||||||
|
manager.submit(model_id)
|
||||||
|
elif operation == 'unload':
|
||||||
|
manager.unload(model_id)
|
||||||
|
elif operation == 'configure':
|
||||||
|
manager.configure(model_id)
|
||||||
|
elif operation != 'status':
|
||||||
|
raise ValueError('模型服务操作无效')
|
||||||
|
return result.success(manager.status(model_id))
|
||||||
|
except (ValueError, TimeoutError) as exc:
|
||||||
|
raise ValidationError(str(exc))
|
||||||
|
|
@ -46,7 +46,10 @@ class Config(dict):
|
||||||
}
|
}
|
||||||
|
|
||||||
def get_debug(self) -> bool:
|
def get_debug(self) -> bool:
|
||||||
return self.get('DEBUG') if 'DEBUG' in self else True
|
value = self.get('DEBUG', True)
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip().lower() in ('1', 'true', 'yes', 'on')
|
||||||
|
return bool(value)
|
||||||
|
|
||||||
def get_time_zone(self) -> str:
|
def get_time_zone(self) -> str:
|
||||||
return self.get('TIME_ZONE') if 'TIME_ZONE' in self else 'Asia/Shanghai'
|
return self.get('TIME_ZONE') if 'TIME_ZONE' in self else 'Asia/Shanghai'
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,7 @@ MIDDLEWARE = [
|
||||||
REST_FRAMEWORK = {
|
REST_FRAMEWORK = {
|
||||||
'EXCEPTION_HANDLER': 'common.exception.handle_exception.handle_exception',
|
'EXCEPTION_HANDLER': 'common.exception.handle_exception.handle_exception',
|
||||||
'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
|
'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
|
||||||
|
'UNAUTHENTICATED_USER': None,
|
||||||
'DEFAULT_AUTHENTICATION_CLASSES': ['common.auth.authenticate.AnonymousAuthentication']
|
'DEFAULT_AUTHENTICATION_CLASSES': ['common.auth.authenticate.AnonymousAuthentication']
|
||||||
}
|
}
|
||||||
STATICFILES_DIRS = [(os.path.join(PROJECT_DIR, 'ui', 'dist'))]
|
STATICFILES_DIRS = [(os.path.join(PROJECT_DIR, 'ui', 'dist'))]
|
||||||
|
|
|
||||||
|
|
@ -97,8 +97,11 @@ class MaxKBBaseModel(ABC):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def filter_optional_params(model_kwargs):
|
def filter_optional_params(model_kwargs):
|
||||||
|
from system_manage.summary_plan import LOCAL_PARAMETERS
|
||||||
optional_params = {}
|
optional_params = {}
|
||||||
for key, value in model_kwargs.items():
|
for key, value in model_kwargs.items():
|
||||||
|
if key in LOCAL_PARAMETERS and key != 'timeout':
|
||||||
|
continue
|
||||||
if key not in ['model_id', 'use_local', 'streaming', 'show_ref_label', 'stream']:
|
if key not in ['model_id', 'use_local', 'streaming', 'show_ref_label', 'stream']:
|
||||||
if key == 'extra_body' and isinstance(value, dict):
|
if key == 'extra_body' and isinstance(value, dict):
|
||||||
optional_params = {**optional_params, **value}
|
optional_params = {**optional_params, **value}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
"""Deployment defaults; explicit model settings always take precedence."""
|
||||||
|
|
||||||
|
DEFAULT_CONTEXT_TOKENS = 65536
|
||||||
|
DEFAULT_SUMMARY_OUTPUT_TOKENS = 8192
|
||||||
|
DEFAULT_SAFETY_TOKENS = 4096
|
||||||
|
|
||||||
|
|
||||||
|
def model_context(params):
|
||||||
|
value = params.get('context_tokens')
|
||||||
|
if value is None:
|
||||||
|
value = params.get('num_ctx', DEFAULT_CONTEXT_TOKENS)
|
||||||
|
if isinstance(value, str) and value.isdigit():
|
||||||
|
value = int(value)
|
||||||
|
if type(value) is not int or not 1024 <= value <= 2000000:
|
||||||
|
raise ValueError('模型上下文须为 1024 至 2000000 的整数')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def summary_defaults(params):
|
||||||
|
result = dict(params)
|
||||||
|
context = model_context(result)
|
||||||
|
result['context_tokens'] = context
|
||||||
|
result.setdefault('safety_tokens', min(DEFAULT_SAFETY_TOKENS, max(256, context // 16)))
|
||||||
|
safety = int(result['safety_tokens'])
|
||||||
|
output = result.get('output_tokens', result.get('max_tokens'))
|
||||||
|
if output is None:
|
||||||
|
output = min(DEFAULT_SUMMARY_OUTPUT_TOKENS, max(256, context // 8),
|
||||||
|
int(result.get('max_output_tokens') or DEFAULT_SUMMARY_OUTPUT_TOKENS))
|
||||||
|
result['output_tokens'] = int(output)
|
||||||
|
result.setdefault('max_output_tokens', max(int(output), min(DEFAULT_SUMMARY_OUTPUT_TOKENS,
|
||||||
|
context - safety - 1024)))
|
||||||
|
result.setdefault('max_input_tokens', context)
|
||||||
|
result.setdefault('input_tokens', min(int(result['max_input_tokens']), context - int(output) - safety))
|
||||||
|
return result
|
||||||
|
|
@ -25,6 +25,20 @@ class BaiLianLLMModelParams(BaseForm):
|
||||||
precision=2
|
precision=2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
top_p = forms.SliderField(
|
||||||
|
TooltipLabel(
|
||||||
|
_('TopP'),
|
||||||
|
_('Nucleus sampling: the model considers the results of the top_p probability mass of tokens. '
|
||||||
|
'The smaller the value, the more deterministic the generation')
|
||||||
|
),
|
||||||
|
required=True,
|
||||||
|
default_value=0.8,
|
||||||
|
_min=0,
|
||||||
|
_max=1,
|
||||||
|
_step=0.01,
|
||||||
|
precision=2
|
||||||
|
)
|
||||||
|
|
||||||
max_tokens = forms.SliderField(
|
max_tokens = forms.SliderField(
|
||||||
TooltipLabel(
|
TooltipLabel(
|
||||||
_('Output the maximum Tokens'),
|
_('Output the maximum Tokens'),
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,9 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
||||||
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
|
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
|
||||||
|
# Client controls belong to the SDK, not the provider's JSON body.
|
||||||
|
client_params = {key: optional_params.pop(key) for key in
|
||||||
|
('temperature', 'top_p', 'max_tokens', 'timeout', 'max_retries') if key in optional_params}
|
||||||
# if 'qwen-omni-turbo' in model_name or 'qwq' in model_name:
|
# if 'qwen-omni-turbo' in model_name or 'qwq' in model_name:
|
||||||
# optional_params['streaming'] = True
|
# optional_params['streaming'] = True
|
||||||
return BaiLianChatModel(
|
return BaiLianChatModel(
|
||||||
|
|
@ -21,5 +24,7 @@ class BaiLianChatModel(MaxKBBaseModel, BaseChatOpenAI):
|
||||||
openai_api_base=model_credential.get('api_base'),
|
openai_api_base=model_credential.get('api_base'),
|
||||||
openai_api_key=model_credential.get('api_key'),
|
openai_api_key=model_credential.get('api_key'),
|
||||||
streaming=True,
|
streaming=True,
|
||||||
|
stream_usage=True,
|
||||||
|
**client_params,
|
||||||
extra_body=optional_params
|
extra_body=optional_params
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _, gettext
|
||||||
|
|
||||||
from common import forms
|
from common import forms
|
||||||
from common.exception.app_exception import AppApiException
|
from common.exception.app_exception import AppApiException
|
||||||
from common.forms import BaseForm
|
from common.forms import BaseForm, TooltipLabel
|
||||||
from models_provider.base_model_provider import BaseModelCredential, ValidCode
|
from models_provider.base_model_provider import BaseModelCredential, ValidCode
|
||||||
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
|
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
|
||||||
|
|
||||||
|
|
@ -51,3 +51,29 @@ class LocalEmbeddingCredential(BaseForm, BaseModelCredential):
|
||||||
return model
|
return model
|
||||||
|
|
||||||
cache_folder = forms.TextInputField(_('Model catalog'), required=True)
|
cache_folder = forms.TextInputField(_('Model catalog'), required=True)
|
||||||
|
device = forms.SingleSelect(
|
||||||
|
TooltipLabel(_('Device'), _('Runtime device for the embedding model')),
|
||||||
|
text_field='label',
|
||||||
|
value_field='value',
|
||||||
|
option_list=[
|
||||||
|
{'value': 'cpu', 'label': 'CPU'},
|
||||||
|
{'value': 'cuda', 'label': 'GPU(CUDA)'},
|
||||||
|
],
|
||||||
|
required=False,
|
||||||
|
default_value='cpu'
|
||||||
|
)
|
||||||
|
normalize = forms.SwitchField(
|
||||||
|
TooltipLabel(_('Normalize'), _('Whether to normalize embeddings to unit length')),
|
||||||
|
required=False,
|
||||||
|
default_value=True,
|
||||||
|
attrs={"active-value": "true", "inactive-value": "false"}
|
||||||
|
)
|
||||||
|
batch_size = forms.SliderField(
|
||||||
|
TooltipLabel(_('Batch size'), _('Number of texts per embedding request')),
|
||||||
|
required=False,
|
||||||
|
default_value=10,
|
||||||
|
_min=1,
|
||||||
|
_max=64,
|
||||||
|
_step=1,
|
||||||
|
precision=0
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ from models_provider.base_model_provider import MaxKBBaseModel
|
||||||
|
|
||||||
|
|
||||||
class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings):
|
class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings):
|
||||||
|
batch_size: int = 10
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def is_cache_model():
|
def is_cache_model():
|
||||||
return True
|
return True
|
||||||
|
|
@ -27,7 +28,15 @@ class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings):
|
||||||
local_model_path = os.path.join(cache_folder, model_name)
|
local_model_path = os.path.join(cache_folder, model_name)
|
||||||
if os.path.isdir(local_model_path):
|
if os.path.isdir(local_model_path):
|
||||||
model_path = local_model_path
|
model_path = local_model_path
|
||||||
return LocalEmbedding(model_name=model_path, cache_folder=cache_folder,
|
normalize = model_credential.get('normalize', True)
|
||||||
|
if isinstance(normalize, str):
|
||||||
|
normalize = normalize.lower() == 'true'
|
||||||
|
instance = LocalEmbedding(model_name=model_path, cache_folder=cache_folder,
|
||||||
model_kwargs={'device': model_credential.get('device')},
|
model_kwargs={'device': model_credential.get('device')},
|
||||||
encode_kwargs={'normalize_embeddings': True}
|
encode_kwargs={'normalize_embeddings': normalize if normalize is not None else True}
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
instance.batch_size = int(model_credential.get('batch_size') or 10)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
instance.batch_size = 10
|
||||||
|
return instance
|
||||||
|
|
|
||||||
|
|
@ -24,19 +24,25 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
||||||
|
try:
|
||||||
|
batch_size = int(model_credential.get('batch_size') or 10)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
batch_size = 10
|
||||||
return LocalEmbedding(model_name=model_name, cache_folder=model_credential.get('cache_folder'),
|
return LocalEmbedding(model_name=model_name, cache_folder=model_credential.get('cache_folder'),
|
||||||
model_kwargs={'device': model_credential.get('device')},
|
model_kwargs={'device': model_credential.get('device')},
|
||||||
encode_kwargs={'normalize_embeddings': True},
|
encode_kwargs={'normalize_embeddings': True},
|
||||||
|
batch_size=batch_size,
|
||||||
**model_kwargs)
|
**model_kwargs)
|
||||||
|
|
||||||
model_id: str = None
|
model_id: str = None
|
||||||
|
batch_size: int = 10
|
||||||
|
|
||||||
def embed_query(self, text: str) -> List[float]:
|
def embed_query(self, text: str) -> List[float]:
|
||||||
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
|
bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}'
|
||||||
prefix = CONFIG.get_admin_path()
|
prefix = CONFIG.get_admin_path()
|
||||||
res = requests.post(
|
res = requests.post(
|
||||||
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_query',
|
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_query',
|
||||||
{'text': text})
|
{'text': text}, timeout=(5, 620))
|
||||||
result = res.json()
|
result = res.json()
|
||||||
if result.get('code', 500) == 200:
|
if result.get('code', 500) == 200:
|
||||||
return result.get('data')
|
return result.get('data')
|
||||||
|
|
@ -47,7 +53,7 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings):
|
||||||
prefix = CONFIG.get_admin_path()
|
prefix = CONFIG.get_admin_path()
|
||||||
res = requests.post(
|
res = requests.post(
|
||||||
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_documents',
|
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_documents',
|
||||||
{'texts': texts})
|
{'texts': texts}, timeout=(5, 620))
|
||||||
result = res.json()
|
result = res.json()
|
||||||
if result.get('code', 500) == 200:
|
if result.get('code', 500) == 200:
|
||||||
return result.get('data')
|
return result.get('data')
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Optional, Dict, Any
|
from typing import Sequence, Optional, Dict, Any
|
||||||
|
import os
|
||||||
|
|
||||||
from langchain_core.callbacks import Callbacks
|
from langchain_core.callbacks import Callbacks
|
||||||
from langchain_core.documents import Document, BaseDocumentCompressor
|
from langchain_core.documents import Document, BaseDocumentCompressor
|
||||||
|
|
@ -39,7 +40,10 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor):
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
||||||
return LocalReranker(model_name, cache_dir=model_credential.get('cache_dir'))
|
cache_dir = model_credential.get('cache_dir') or model_credential.get('cache_folder')
|
||||||
|
if cache_dir and not os.path.isabs(model_name) and os.path.isdir(os.path.join(cache_dir, model_name)):
|
||||||
|
model_name = os.path.join(cache_dir, model_name)
|
||||||
|
return LocalReranker(model_name, cache_dir=cache_dir, device=model_credential.get('device') or 'cpu')
|
||||||
|
|
||||||
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
|
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
|
||||||
Sequence[Document]:
|
Sequence[Document]:
|
||||||
|
|
@ -48,7 +52,7 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor):
|
||||||
import torch
|
import torch
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
inputs = self.tokenizer([[query, document.page_content] for document in documents], padding=True,
|
inputs = self.tokenizer([[query, document.page_content] for document in documents], padding=True,
|
||||||
truncation=True, return_tensors='pt', max_length=512)
|
truncation=True, return_tensors='pt', max_length=512).to(self.client.device)
|
||||||
scores = [torch.sigmoid(s).float().item() for s in
|
scores = [torch.sigmoid(s).float().item() for s in
|
||||||
self.client(**inputs, return_dict=True).logits.view(-1, ).float()]
|
self.client(**inputs, return_dict=True).logits.view(-1, ).float()]
|
||||||
result = [Document(page_content=documents[index].page_content, metadata={'relevance_score': scores[index]})
|
result = [Document(page_content=documents[index].page_content, metadata={'relevance_score': scores[index]})
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,6 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor):
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
print('ssss', kwargs.get('model_id', None))
|
|
||||||
self.model_id = kwargs.get('model_id', None)
|
self.model_id = kwargs.get('model_id', None)
|
||||||
|
|
||||||
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
|
def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \
|
||||||
|
|
@ -44,7 +43,7 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor):
|
||||||
res = requests.post(
|
res = requests.post(
|
||||||
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/compress_documents',
|
f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/compress_documents',
|
||||||
json={'documents': [{'page_content': document.page_content, 'metadata': document.metadata} for document in
|
json={'documents': [{'page_content': document.page_content, 'metadata': document.metadata} for document in
|
||||||
documents], 'query': query}, headers={'Content-Type': 'application/json'})
|
documents], 'query': query}, headers={'Content-Type': 'application/json'}, timeout=(5, 620))
|
||||||
result = res.json()
|
result = res.json()
|
||||||
if result.get('code', 500) == 200:
|
if result.get('code', 500) == 200:
|
||||||
return [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document
|
return [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ from django.utils.translation import gettext as _
|
||||||
|
|
||||||
from common import forms
|
from common import forms
|
||||||
from common.exception.app_exception import AppApiException
|
from common.exception.app_exception import AppApiException
|
||||||
from common.forms import BaseForm
|
from common.forms import BaseForm, TooltipLabel
|
||||||
from models_provider.base_model_provider import BaseModelCredential, ValidCode
|
from models_provider.base_model_provider import BaseModelCredential, ValidCode
|
||||||
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
|
from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding
|
||||||
|
|
||||||
|
|
@ -47,3 +47,12 @@ class OllamaEmbeddingModelCredential(BaseForm, BaseModelCredential):
|
||||||
return self
|
return self
|
||||||
|
|
||||||
api_base = forms.TextInputField('API URL', required=True)
|
api_base = forms.TextInputField('API URL', required=True)
|
||||||
|
batch_size = forms.SliderField(
|
||||||
|
TooltipLabel(_('Batch size'), _('Number of texts per embedding request')),
|
||||||
|
required=False,
|
||||||
|
default_value=10,
|
||||||
|
_min=1,
|
||||||
|
_max=64,
|
||||||
|
_step=1,
|
||||||
|
precision=0
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -14,12 +14,26 @@ from models_provider.base_model_provider import MaxKBBaseModel
|
||||||
|
|
||||||
|
|
||||||
class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
|
class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
|
||||||
|
model_resource_id: str | None = None
|
||||||
|
runtime_keep_alive: int = 300
|
||||||
|
batch_size: int = 10
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _default_params(self):
|
||||||
|
return {**super()._default_params, 'keep_alive': self.runtime_keep_alive}
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs):
|
||||||
return OllamaEmbedding(
|
try:
|
||||||
|
batch_size = int(model_credential.get('batch_size') or 10)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
batch_size = 10
|
||||||
|
instance = OllamaEmbedding(
|
||||||
model=model_name,
|
model=model_name,
|
||||||
base_url=model_credential.get('api_base'),
|
base_url=model_credential.get('api_base'),
|
||||||
|
model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None,
|
||||||
)
|
)
|
||||||
|
instance.batch_size = batch_size
|
||||||
|
return instance
|
||||||
|
|
||||||
def embed_documents(self, texts: List[str]) -> List[List[float]]:
|
def embed_documents(self, texts: List[str]) -> List[List[float]]:
|
||||||
"""Embed documents using an Ollama deployed embedding model.
|
"""Embed documents using an Ollama deployed embedding model.
|
||||||
|
|
@ -31,8 +45,13 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
|
||||||
List of embeddings, one for each text.
|
List of embeddings, one for each text.
|
||||||
"""
|
"""
|
||||||
instruction_pairs = [f"{text}" for text in texts]
|
instruction_pairs = [f"{text}" for text in texts]
|
||||||
embeddings = self._embed(instruction_pairs)
|
if not texts:
|
||||||
return embeddings
|
return []
|
||||||
|
from common.model_runtime_client import inference_lease
|
||||||
|
with inference_lease(self.model_resource_id) as state:
|
||||||
|
if state:
|
||||||
|
self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
|
||||||
|
return self._embed(instruction_pairs)
|
||||||
|
|
||||||
def embed_query(self, text: str) -> List[float]:
|
def embed_query(self, text: str) -> List[float]:
|
||||||
"""Embed a query using a Ollama deployed embedding model.
|
"""Embed a query using a Ollama deployed embedding model.
|
||||||
|
|
@ -44,5 +63,8 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings):
|
||||||
Embeddings for the text.
|
Embeddings for the text.
|
||||||
"""
|
"""
|
||||||
instruction_pair = f"{text}"
|
instruction_pair = f"{text}"
|
||||||
embedding = self._embed([instruction_pair])[0]
|
from common.model_runtime_client import inference_lease
|
||||||
return embedding
|
with inference_lease(self.model_resource_id) as state:
|
||||||
|
if state:
|
||||||
|
self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
|
||||||
|
return self._embed([instruction_pair])[0]
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ def get_base_url(url: str):
|
||||||
|
|
||||||
|
|
||||||
class OllamaChatModel(MaxKBBaseModel, ChatOllama):
|
class OllamaChatModel(MaxKBBaseModel, ChatOllama):
|
||||||
|
model_resource_id: str | None = None
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def is_cache_model():
|
def is_cache_model():
|
||||||
return False
|
return False
|
||||||
|
|
@ -35,10 +36,28 @@ class OllamaChatModel(MaxKBBaseModel, ChatOllama):
|
||||||
api_base = model_credential.get('api_base', '')
|
api_base = model_credential.get('api_base', '')
|
||||||
base_url = get_base_url(api_base)
|
base_url = get_base_url(api_base)
|
||||||
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
|
optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs)
|
||||||
|
from models_provider.context_policy import model_context
|
||||||
|
optional_params['num_ctx'] = model_context(model_kwargs)
|
||||||
|
|
||||||
return OllamaChatModel(model=model_name, base_url=base_url,
|
return OllamaChatModel(model=model_name, base_url=base_url,
|
||||||
|
model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None,
|
||||||
stream=True, **optional_params)
|
stream=True, **optional_params)
|
||||||
|
|
||||||
|
def _create_chat_stream(self, messages, stop=None, **kwargs):
|
||||||
|
from common.model_runtime_client import inference_lease
|
||||||
|
with inference_lease(self.model_resource_id) as state:
|
||||||
|
if state:
|
||||||
|
kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
|
||||||
|
yield from super()._create_chat_stream(messages, stop, **kwargs)
|
||||||
|
|
||||||
|
async def _acreate_chat_stream(self, messages, stop=None, **kwargs):
|
||||||
|
from common.model_runtime_client import async_inference_lease
|
||||||
|
async with async_inference_lease(self.model_resource_id) as state:
|
||||||
|
if state:
|
||||||
|
kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds']
|
||||||
|
async for part in super()._acreate_chat_stream(messages, stop, **kwargs):
|
||||||
|
yield part
|
||||||
|
|
||||||
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
|
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
|
||||||
tokenizer = TokenizerManage.get_tokenizer()
|
tokenizer = TokenizerManage.get_tokenizer()
|
||||||
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])
|
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,258 @@
|
||||||
|
"""Model file assets, independent of runtime/model connection records. Never executes model code."""
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import tempfile
|
||||||
|
import uuid
|
||||||
|
import zipfile
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
MAX_BYTES = 2 * 1024 ** 3
|
||||||
|
MAX_FILES = 10000
|
||||||
|
WEIGHTS = {'.safetensors', '.bin', '.onnx', '.gguf', '.pt', '.pth'}
|
||||||
|
|
||||||
|
|
||||||
|
class ImportError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def model_fingerprint(directory):
|
||||||
|
"""Hash relative paths and complete file contents, independent of the archive name."""
|
||||||
|
root = Path(directory)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for file in sorted((p for p in root.rglob('*') if p.is_file()), key=lambda p: p.relative_to(root).as_posix()):
|
||||||
|
name = file.relative_to(root).as_posix().encode('utf-8')
|
||||||
|
digest.update(len(name).to_bytes(8, 'big'))
|
||||||
|
digest.update(name)
|
||||||
|
digest.update(file.stat().st_size.to_bytes(8, 'big'))
|
||||||
|
with file.open('rb') as source:
|
||||||
|
for chunk in iter(lambda: source.read(1024 * 1024), b''):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def import_lock(root):
|
||||||
|
"""Serialize duplicate checks and record creation across Web processes."""
|
||||||
|
with (root / '.import.lock').open('a+b') as lock:
|
||||||
|
if os.name == 'nt':
|
||||||
|
import msvcrt
|
||||||
|
lock.seek(0)
|
||||||
|
msvcrt.locking(lock.fileno(), msvcrt.LK_LOCK, 1)
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
if os.name == 'nt':
|
||||||
|
lock.seek(0)
|
||||||
|
msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
|
||||||
|
else:
|
||||||
|
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_model(directory):
|
||||||
|
root = Path(directory)
|
||||||
|
if not root.is_dir() or root.is_symlink():
|
||||||
|
raise ImportError('请选择有效模型目录')
|
||||||
|
files = []
|
||||||
|
for current, dirs, names in os.walk(root, followlinks=False):
|
||||||
|
for name in dirs + names:
|
||||||
|
if (Path(current) / name).is_symlink():
|
||||||
|
raise ImportError('模型目录不能包含符号链接')
|
||||||
|
for name in names:
|
||||||
|
file = Path(current) / name
|
||||||
|
if not file.is_file():
|
||||||
|
raise ImportError('模型目录包含非普通文件')
|
||||||
|
files.append(file)
|
||||||
|
if len(files) > MAX_FILES:
|
||||||
|
raise ImportError('模型文件数量超过限制')
|
||||||
|
weights = [f for f in files if f.suffix.lower() in WEIGHTS and f.stat().st_size > 0]
|
||||||
|
if not weights:
|
||||||
|
raise ImportError('未找到有效模型权重文件')
|
||||||
|
|
||||||
|
def read_json(name):
|
||||||
|
file = root / name
|
||||||
|
if not file.exists():
|
||||||
|
return {}
|
||||||
|
if file.stat().st_size > 2 * 1024 ** 2:
|
||||||
|
raise ImportError(name + '超过大小限制')
|
||||||
|
try:
|
||||||
|
data = json.loads(file.read_text(encoding='utf-8'))
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
raise ValueError()
|
||||||
|
return data
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
raise ImportError(name + '格式错误')
|
||||||
|
|
||||||
|
config = read_json('config.json')
|
||||||
|
manifest = read_json('model-manifest.json')
|
||||||
|
quant = config.get('quantization_config') or {}
|
||||||
|
if not isinstance(quant, dict):
|
||||||
|
raise ImportError('量化配置格式错误')
|
||||||
|
quantization = str(quant.get('quant_method', '')).lower()
|
||||||
|
suffixes = {f.suffix.lower() for f in weights}
|
||||||
|
if config and suffixes & {'.safetensors', '.bin'}:
|
||||||
|
if not config.get('model_type') and not config.get('architectures'):
|
||||||
|
raise ImportError('config.json缺少模型架构信息')
|
||||||
|
format_name = {'gptq': 'GPTQ', 'awq': 'AWQ'}.get(quantization, 'HF')
|
||||||
|
elif '.onnx' in suffixes:
|
||||||
|
format_name = 'ONNX'
|
||||||
|
elif '.gguf' in suffixes:
|
||||||
|
format_name = 'GGUF'
|
||||||
|
elif manifest:
|
||||||
|
format_name = 'CUSTOM'
|
||||||
|
else:
|
||||||
|
raise ImportError('模型目录缺少config.json或model-manifest.json')
|
||||||
|
# Validate referenced shards without trusting paths supplied by model files.
|
||||||
|
for index in root.glob('*.index.json'):
|
||||||
|
mapping = read_json(index.name).get('weight_map')
|
||||||
|
if not isinstance(mapping, dict) or not mapping:
|
||||||
|
raise ImportError('权重索引格式错误')
|
||||||
|
for name in mapping.values():
|
||||||
|
if not isinstance(name, str) or '\\' in name:
|
||||||
|
raise ImportError('权重分片路径无效')
|
||||||
|
relative = PurePosixPath(name)
|
||||||
|
if relative.is_absolute() or '..' in relative.parts or not (root / name).is_file():
|
||||||
|
raise ImportError('权重分片缺失或路径无效')
|
||||||
|
return {
|
||||||
|
'format': format_name, 'size_bytes': sum(f.stat().st_size for f in files),
|
||||||
|
'file_count': len(files), 'weight_count': len(weights),
|
||||||
|
'metadata': {key: config.get(key) for key in ('model_type', 'architectures', 'transformers_version', 'torch_dtype') if key in config},
|
||||||
|
'quantization': quantization,
|
||||||
|
'checks': ['权重文件存在', '配置JSON及分片引用检查通过'],
|
||||||
|
'runtime_status': 'NOT_CHECKED',
|
||||||
|
'runtime_note': '文件已导入,运行能力需接入验证',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ModelImportStore:
|
||||||
|
def __init__(self, root):
|
||||||
|
self.root = Path(root)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
def _records(self):
|
||||||
|
records = []
|
||||||
|
for path in self.root.glob('*/import.json'):
|
||||||
|
if path.parent.name.startswith('.import-'):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
records.append(json.loads(path.read_text(encoding='utf-8')))
|
||||||
|
except (ValueError, OSError):
|
||||||
|
continue
|
||||||
|
return sorted(records, key=lambda r: r.get('created_at', ''), reverse=True)
|
||||||
|
|
||||||
|
def list(self):
|
||||||
|
records, paths, fingerprints = [], set(), set()
|
||||||
|
for record in self._records():
|
||||||
|
path = record.get('path')
|
||||||
|
canonical = os.path.normcase(os.path.realpath(path)) if path else None
|
||||||
|
fingerprint = record.get('fingerprint') if record.get('source') == 'UPLOAD' else None
|
||||||
|
if (canonical and canonical in paths) or (fingerprint and fingerprint in fingerprints):
|
||||||
|
continue
|
||||||
|
if canonical:
|
||||||
|
paths.add(canonical)
|
||||||
|
if fingerprint:
|
||||||
|
fingerprints.add(fingerprint)
|
||||||
|
records.append(record)
|
||||||
|
if len(records) == 200:
|
||||||
|
break
|
||||||
|
return records
|
||||||
|
|
||||||
|
def create(self, owner_id, path=None, upload=None):
|
||||||
|
if (path is None) == (upload is None):
|
||||||
|
raise ImportError('请选择一种导入方式')
|
||||||
|
identity = str(uuid.uuid4())
|
||||||
|
staging = Path(tempfile.mkdtemp(prefix='.import-', dir=self.root))
|
||||||
|
try:
|
||||||
|
if upload is not None:
|
||||||
|
if not upload.name.lower().endswith('.zip') or upload.size > MAX_BYTES:
|
||||||
|
raise ImportError('请上传2GB以内的ZIP模型包')
|
||||||
|
archive = staging / 'upload.zip'
|
||||||
|
with archive.open('wb') as output:
|
||||||
|
size = 0
|
||||||
|
for chunk in upload.chunks():
|
||||||
|
size += len(chunk)
|
||||||
|
if size > MAX_BYTES:
|
||||||
|
raise ImportError('模型包超过2GB')
|
||||||
|
output.write(chunk)
|
||||||
|
source = staging / 'files'
|
||||||
|
source.mkdir()
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(archive) as zipped:
|
||||||
|
members = zipped.infolist()
|
||||||
|
if len(members) > MAX_FILES or sum(i.file_size for i in members) > MAX_BYTES:
|
||||||
|
raise ImportError('解压后模型超过2GB或文件数量限制,请使用服务器目录')
|
||||||
|
seen = set()
|
||||||
|
for member in members:
|
||||||
|
name = PurePosixPath(member.filename)
|
||||||
|
mode = member.external_attr >> 16
|
||||||
|
if (name.is_absolute() or '..' in name.parts or '\\' in member.filename
|
||||||
|
or ':' in member.filename or stat.S_ISLNK(mode) or member.flag_bits & 1
|
||||||
|
or member.filename in seen):
|
||||||
|
raise ImportError('模型包包含不安全的路径或文件')
|
||||||
|
seen.add(member.filename)
|
||||||
|
target = source.joinpath(*name.parts)
|
||||||
|
if member.is_dir():
|
||||||
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
|
else:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with zipped.open(member) as incoming, target.open('xb') as output:
|
||||||
|
shutil.copyfileobj(incoming, output, 1024 * 1024)
|
||||||
|
except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
|
||||||
|
raise ImportError('模型包损坏或无法解压') from exc
|
||||||
|
archive.unlink()
|
||||||
|
# Most model archives contain one enclosing directory.
|
||||||
|
children = list(source.iterdir())
|
||||||
|
model_dir = children[0] if len(children) == 1 and children[0].is_dir() else source
|
||||||
|
info = inspect_model(model_dir)
|
||||||
|
final_path = self.root / identity / model_dir.relative_to(staging)
|
||||||
|
source_type = 'UPLOAD'
|
||||||
|
display_name = Path(upload.name).stem
|
||||||
|
else:
|
||||||
|
if not isinstance(path, str) or not path.strip() or not Path(path).is_absolute():
|
||||||
|
raise ImportError('请输入服务器绝对目录')
|
||||||
|
model_dir = Path(path).resolve(strict=True)
|
||||||
|
info = inspect_model(model_dir)
|
||||||
|
final_path, source_type, display_name = model_dir, 'PATH', model_dir.name
|
||||||
|
with import_lock(self.root):
|
||||||
|
existing = self._records()
|
||||||
|
if source_type == 'PATH':
|
||||||
|
canonical = os.path.normcase(os.path.realpath(model_dir))
|
||||||
|
for record in existing:
|
||||||
|
if record.get('path') and os.path.normcase(os.path.realpath(record['path'])) == canonical:
|
||||||
|
return {**record, 'already_imported': True}
|
||||||
|
fingerprint = model_fingerprint(model_dir)
|
||||||
|
for record in existing:
|
||||||
|
if (record.get('size_bytes') != info['size_bytes'] or
|
||||||
|
record.get('file_count') != info['file_count']):
|
||||||
|
continue
|
||||||
|
previous_path = record.get('path')
|
||||||
|
if not previous_path or not Path(previous_path).is_dir():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
inspect_model(previous_path)
|
||||||
|
previous = model_fingerprint(previous_path)
|
||||||
|
except (OSError, ImportError):
|
||||||
|
continue
|
||||||
|
if previous == fingerprint:
|
||||||
|
return {**record, 'already_imported': True}
|
||||||
|
record = {
|
||||||
|
'id': identity, 'name': display_name, 'path': str(final_path),
|
||||||
|
'source': source_type, 'owner_id': str(owner_id),
|
||||||
|
'created_at': datetime.now(timezone.utc).isoformat(),
|
||||||
|
'status': 'IMPORTED', 'fingerprint': fingerprint, **info,
|
||||||
|
}
|
||||||
|
(staging / 'import.json').write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8')
|
||||||
|
staging.rename(self.root / identity)
|
||||||
|
return record
|
||||||
|
except (FileNotFoundError, PermissionError) as exc:
|
||||||
|
raise ImportError('目录不存在或无访问权限') from exc
|
||||||
|
finally:
|
||||||
|
if staging.exists():
|
||||||
|
shutil.rmtree(staging)
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""补齐已有百炼模型表单中的 TopP;保留全部自定义参数项。"""
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
TOP_P_FIELD = {
|
||||||
|
"input_type": "Slider", "label": "TopP", "required": True,
|
||||||
|
"default_value": 0.8, "relation_show_field_dict": {},
|
||||||
|
"relation_trigger_field_dict": {}, "trigger_type": "OPTION_LIST",
|
||||||
|
"attrs": {"min": 0, "max": 1, "step": 0.01, "precision": 2,
|
||||||
|
"show-input-controls": False, "show-input": True},
|
||||||
|
"props_info": {}, "field": "top_p",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add_top_p_params_form(apps, schema_editor):
|
||||||
|
Model = apps.get_model('models_provider', 'Model')
|
||||||
|
for model in Model.objects.using(schema_editor.connection.alias).filter(
|
||||||
|
provider='aliyun_bai_lian_model_provider', model_type='LLM'):
|
||||||
|
form = model.model_params_form or []
|
||||||
|
if any(item.get('field') == 'top_p' for item in form):
|
||||||
|
continue
|
||||||
|
model.model_params_form = [*form, TOP_P_FIELD]
|
||||||
|
model.save(update_fields=['model_params_form'], using=schema_editor.connection.alias)
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [('models_provider', '0001_initial')]
|
||||||
|
# 回退代码时保留参数项,不删除可能由管理员维护的数据。
|
||||||
|
operations = [migrations.RunPython(add_top_p_params_form, migrations.RunPython.noop)]
|
||||||
|
|
@ -15,5 +15,5 @@ FROM (SELECT model."id"::text, model."name",
|
||||||
from model
|
from model
|
||||||
left join "user" on user_id = "user".id
|
left join "user" on user_id = "user".id
|
||||||
where model."id"::text in (select target
|
where model."id"::text in (select target
|
||||||
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
|
from effective_resource_permission ${workspace_user_resource_permission_query_set}
|
||||||
and 'VIEW' = any (permission_list)) ) temp ${model_query_set}
|
and 'VIEW' = any (permission_list)) ) temp ${model_query_set}
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ FROM (SELECT model."id"::text, model."name",
|
||||||
from model
|
from model
|
||||||
left join "user" on user_id = "user".id
|
left join "user" on user_id = "user".id
|
||||||
where model."id"::text in (select target
|
where model."id"::text in (select target
|
||||||
from workspace_user_resource_permission ${workspace_user_resource_permission_query_set}
|
from effective_resource_permission ${workspace_user_resource_permission_query_set}
|
||||||
and case
|
and case
|
||||||
when auth_type = 'ROLE' then
|
when auth_type = 'ROLE' then
|
||||||
'ROLE' = any (permission_list)
|
'ROLE' = any (permission_list)
|
||||||
|
|
@ -27,9 +27,9 @@ FROM (SELECT model."id"::text, model."name",
|
||||||
from role_permission role_permission
|
from role_permission role_permission
|
||||||
right join user_role_relation user_role_relation
|
right join user_role_relation user_role_relation
|
||||||
on user_role_relation.role_id = role_permission.role_id
|
on user_role_relation.role_id = role_permission.role_id
|
||||||
where user_role_relation.user_id = workspace_user_resource_permission.user_id
|
where user_role_relation.user_id = effective_resource_permission.user_id
|
||||||
and user_role_relation.workspace_id =
|
and user_role_relation.workspace_id =
|
||||||
workspace_user_resource_permission.workspace_id)
|
effective_resource_permission.workspace_id)
|
||||||
|
|
||||||
else
|
else
|
||||||
'VIEW' = any (permission_list)
|
'VIEW' = any (permission_list)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,74 @@
|
||||||
|
"""Context inheritance and native provider options; no network or database."""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import TestCase
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from models_provider.context_policy import model_context, summary_defaults
|
||||||
|
|
||||||
|
|
||||||
|
class ContextPolicyTests(TestCase):
|
||||||
|
def test_platform_budget_is_64k_with_room_for_output_and_prompt(self):
|
||||||
|
from system_manage.ai_hub_organize import resolve_settings
|
||||||
|
defaults = summary_defaults({})
|
||||||
|
self.assertEqual(defaults, {'context_tokens': 65536, 'max_input_tokens': 65536,
|
||||||
|
'input_tokens': 53248, 'max_output_tokens': 8192, 'output_tokens': 8192, 'safety_tokens': 4096})
|
||||||
|
self.assertEqual(resolve_settings({}, {}),
|
||||||
|
{'context_tokens': 65536, 'output_tokens': 8192, 'safety_tokens': 4096})
|
||||||
|
|
||||||
|
def test_explicit_model_limits_and_summary_budgets_win(self):
|
||||||
|
custom = {'context_tokens': '1000000', 'max_input_tokens': '1000000', 'input_tokens': '131072',
|
||||||
|
'max_output_tokens': '384000', 'output_tokens': '16384', 'safety_tokens': '4096'}
|
||||||
|
defaults = summary_defaults(custom)
|
||||||
|
self.assertEqual(defaults['context_tokens'], 1000000)
|
||||||
|
for key in custom.keys() - {'context_tokens', 'output_tokens'}:
|
||||||
|
self.assertEqual(defaults[key], custom[key])
|
||||||
|
self.assertEqual(defaults['output_tokens'], 16384)
|
||||||
|
self.assertEqual(model_context({'num_ctx': '131072'}), 131072)
|
||||||
|
|
||||||
|
def test_smaller_model_and_maximum_output_are_respected(self):
|
||||||
|
small = summary_defaults({'context_tokens': 4096, 'max_output_tokens': 256})
|
||||||
|
self.assertEqual(small['output_tokens'], 256)
|
||||||
|
self.assertLess(small['input_tokens'] + small['output_tokens'], 4096)
|
||||||
|
for value in (True, 0, -1, 1023, 2000001, 'invalid'):
|
||||||
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||||
|
model_context({'context_tokens': value})
|
||||||
|
|
||||||
|
def test_only_llm_gets_default_context_from_model_configuration(self):
|
||||||
|
from models_provider.tools import get_model_default_params
|
||||||
|
model = SimpleNamespace(model_type='LLM', model_params_form=[])
|
||||||
|
self.assertEqual(get_model_default_params(model), {'context_tokens': 65536})
|
||||||
|
model.model_params_form = [{'field': 'num_ctx', 'default_value': '131072'}]
|
||||||
|
self.assertEqual(get_model_default_params(model)['context_tokens'], 131072)
|
||||||
|
model.model_type = 'EMBEDDING'
|
||||||
|
model.model_params_form = []
|
||||||
|
self.assertEqual(get_model_default_params(model), {})
|
||||||
|
|
||||||
|
def test_ollama_native_context_defaults_and_explicit_override(self):
|
||||||
|
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
|
||||||
|
for params, expected in (({}, 65536), ({'context_tokens': 131072}, 131072),
|
||||||
|
({'num_ctx': 32768}, 32768),
|
||||||
|
({'context_tokens': 131072, 'num_ctx': 32768}, 131072)):
|
||||||
|
with patch.object(OllamaChatModel, '__new__', return_value=SimpleNamespace()) as constructor:
|
||||||
|
OllamaChatModel.new_instance('LLM', 'local', {'api_base': 'http://localhost:11434'}, **params)
|
||||||
|
self.assertEqual(constructor.call_args.kwargs['num_ctx'], expected)
|
||||||
|
self.assertNotIn('context_tokens', constructor.call_args.kwargs)
|
||||||
|
|
||||||
|
def test_native_application_override_is_not_hidden_by_platform_default(self):
|
||||||
|
from models_provider import tools
|
||||||
|
model = SimpleNamespace(model_type='LLM', model_params_form=[])
|
||||||
|
with patch.object(tools, 'get_model_by_id', return_value=model), \
|
||||||
|
patch.object(tools.ModelManage, 'get_model', side_effect=lambda key, factory: factory(key)), \
|
||||||
|
patch.object(tools, 'get_model') as create:
|
||||||
|
tools.get_model_instance_by_model_workspace_id('test', 'default', num_ctx=131072)
|
||||||
|
self.assertEqual(create.call_args.kwargs['context_tokens'], 131072)
|
||||||
|
|
||||||
|
def test_metadata_and_preview_share_effective_budget_without_model_calls(self):
|
||||||
|
from system_manage.mcp_business import summary_model_metadata
|
||||||
|
from models_provider.tools import get_model_default_params
|
||||||
|
model = SimpleNamespace(id='test', name='test', model_type='LLM', provider='model_ollama_provider',
|
||||||
|
model_params_form=[])
|
||||||
|
with patch('models_provider.tools.get_model') as invoke:
|
||||||
|
metadata = summary_model_metadata(model)
|
||||||
|
invoke.assert_not_called()
|
||||||
|
self.assertEqual(metadata['summary_defaults']['input_tokens'], 53248)
|
||||||
|
self.assertEqual(metadata['summary_limits']['context_tokens'], 65536)
|
||||||
|
|
@ -0,0 +1,158 @@
|
||||||
|
"""File import regression tests, temporary files only; no database or model runtimes."""
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
from models_provider.local_imports import ModelImportStore, inspect_model, ImportError
|
||||||
|
|
||||||
|
|
||||||
|
class Upload:
|
||||||
|
def __init__(self, entries):
|
||||||
|
data = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(data, 'w') as archive:
|
||||||
|
for name, content in entries:
|
||||||
|
archive.writestr(name, content)
|
||||||
|
self.body = data.getvalue()
|
||||||
|
self.size, self.name = len(self.body), 'model.zip'
|
||||||
|
|
||||||
|
def chunks(self):
|
||||||
|
yield self.body
|
||||||
|
|
||||||
|
|
||||||
|
class ImportTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temp.cleanup)
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
self.model = self.root / 'model'
|
||||||
|
self.model.mkdir()
|
||||||
|
(self.model / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
|
||||||
|
(self.model / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
|
||||||
|
self.store = ModelImportStore(self.root / 'imports')
|
||||||
|
|
||||||
|
def test_local_path_record_persists_without_copying_or_running(self):
|
||||||
|
record = self.store.create('owner', path=str(self.model))
|
||||||
|
self.assertEqual(record['format'], 'HF')
|
||||||
|
self.assertEqual(record['runtime_status'], 'NOT_CHECKED')
|
||||||
|
self.assertEqual(record['path'], str(self.model.resolve()))
|
||||||
|
self.assertEqual(ModelImportStore(self.store.root).list(), [record])
|
||||||
|
|
||||||
|
def test_quantized_formats(self):
|
||||||
|
for method in ('gptq', 'awq'):
|
||||||
|
(self.model / 'config.json').write_text(json.dumps({'model_type': 'qwen2', 'quantization_config': {'quant_method': method}}))
|
||||||
|
self.assertEqual(inspect_model(self.model)['format'], method.upper())
|
||||||
|
|
||||||
|
def test_onnx_gguf_custom(self):
|
||||||
|
(self.model / 'config.json').unlink()
|
||||||
|
(self.model / 'model.safetensors').unlink()
|
||||||
|
for suffix, expected in (('.onnx', 'ONNX'), ('.gguf', 'GGUF'), ('.pth', 'CUSTOM')):
|
||||||
|
file = self.model / ('model' + suffix)
|
||||||
|
file.write_bytes(b'fixture')
|
||||||
|
(self.model / 'model-manifest.json').write_text('{"version":"1"}')
|
||||||
|
self.assertEqual(inspect_model(self.model)['format'], expected)
|
||||||
|
file.unlink()
|
||||||
|
|
||||||
|
def test_zip_nested_model_and_persisted_path(self):
|
||||||
|
record = self.store.create('owner', upload=Upload([('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]))
|
||||||
|
self.assertTrue(Path(record['path'], 'model.safetensors').exists())
|
||||||
|
self.assertEqual(record['source'], 'UPLOAD')
|
||||||
|
|
||||||
|
def test_traversal_windows_path_rejected(self):
|
||||||
|
for name in ('../escape.bin', '/escape.bin', 'C:/escape.bin', 'repo\\escape.bin'):
|
||||||
|
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([(name, b'x')]))
|
||||||
|
self.assertEqual(self.store.list(), [])
|
||||||
|
|
||||||
|
def test_duplicate_archive_entry_rejected(self):
|
||||||
|
with self.assertRaises(ImportError): self.store.create('owner', upload=Upload([('a.bin', b'x'), ('a.bin', b'y')]))
|
||||||
|
|
||||||
|
def test_zip_symlink_rejected(self):
|
||||||
|
data=io.BytesIO()
|
||||||
|
with zipfile.ZipFile(data,'w') as archive:
|
||||||
|
info=zipfile.ZipInfo('model.bin'); info.create_system=3; info.external_attr=(0o120777 << 16)
|
||||||
|
archive.writestr(info, '../outside')
|
||||||
|
upload=Upload([]);upload.body=data.getvalue();upload.size=len(upload.body)
|
||||||
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||||
|
|
||||||
|
def test_size_limit_and_cleanup(self):
|
||||||
|
upload=Upload([('model.onnx', b'x'*100)])
|
||||||
|
with patch('models_provider.local_imports.MAX_BYTES', 20):
|
||||||
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||||
|
self.assertEqual(list(self.store.root.iterdir()), [])
|
||||||
|
|
||||||
|
def test_bad_json_missing_weights_and_unknown_folder_rejected(self):
|
||||||
|
(self.model / 'config.json').write_text('{broken')
|
||||||
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||||
|
(self.model / 'config.json').write_text('{"other":1}')
|
||||||
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||||
|
(self.model / 'model.safetensors').unlink()
|
||||||
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||||
|
|
||||||
|
def test_missing_or_escaping_shard_rejected(self):
|
||||||
|
for name in ('absent.safetensors','../outside.bin'):
|
||||||
|
(self.model / 'model.safetensors.index.json').write_text(json.dumps({'weight_map': {'a':name}}))
|
||||||
|
with self.assertRaises(ImportError):inspect_model(self.model)
|
||||||
|
|
||||||
|
def test_bad_zip_and_missing_path_leave_no_records(self):
|
||||||
|
upload=Upload([]);upload.body=b'not zip';upload.size=7
|
||||||
|
with self.assertRaises(ImportError):self.store.create('owner',upload=upload)
|
||||||
|
with self.assertRaises(ImportError):self.store.create('owner',path=str(self.root/'absent'))
|
||||||
|
self.assertEqual(self.store.list(), [])
|
||||||
|
|
||||||
|
def test_exclusive_source(self):
|
||||||
|
for kwargs in ({}, {'path':str(self.model),'upload':Upload([])}):
|
||||||
|
with self.assertRaises(ImportError):self.store.create('owner',**kwargs)
|
||||||
|
|
||||||
|
def test_repeated_path_and_historical_records_are_deduplicated(self):
|
||||||
|
first = self.store.create('owner', path=str(self.model))
|
||||||
|
repeated = self.store.create('owner', path=str(self.model / '..' / 'model'))
|
||||||
|
self.assertEqual(repeated['id'], first['id'])
|
||||||
|
self.assertTrue(repeated['already_imported'])
|
||||||
|
legacy = self.store.root / 'legacy'
|
||||||
|
legacy.mkdir()
|
||||||
|
(legacy / 'import.json').write_text(json.dumps({**first, 'id': 'legacy', 'created_at': '2000'}))
|
||||||
|
self.assertEqual([row['id'] for row in self.store.list()], [first['id']])
|
||||||
|
|
||||||
|
def test_zip_content_deduplicates_and_distinct_content_is_retained(self):
|
||||||
|
files = [('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights')]
|
||||||
|
first = self.store.create('owner', upload=Upload(files))
|
||||||
|
second = self.store.create('owner', upload=Upload(list(reversed(files))))
|
||||||
|
self.assertEqual(first['id'], second['id'])
|
||||||
|
self.assertTrue(second['already_imported'])
|
||||||
|
different = self.store.create('owner', upload=Upload([
|
||||||
|
('repo/config.json', '{"model_type":"bert"}'), ('repo/model.safetensors', b'weights2')]))
|
||||||
|
self.assertNotEqual(first['id'], different['id'])
|
||||||
|
self.assertEqual(len(self.store.list()), 2)
|
||||||
|
|
||||||
|
def test_different_directory_and_zip_with_same_files_reuse_record(self):
|
||||||
|
first = self.store.create('owner', path=str(self.model))
|
||||||
|
copy = self.root / 'copy'
|
||||||
|
import shutil
|
||||||
|
shutil.copytree(self.model, copy)
|
||||||
|
second = self.store.create('owner', path=str(copy))
|
||||||
|
self.assertEqual(first['id'], second['id'])
|
||||||
|
zipped = Upload([('repo/config.json', (copy / 'config.json').read_bytes()),
|
||||||
|
('repo/model.safetensors', (copy / 'model.safetensors').read_bytes())])
|
||||||
|
self.assertEqual(self.store.create('owner', upload=zipped)['id'], first['id'])
|
||||||
|
|
||||||
|
def test_concurrent_imports_create_one_record(self):
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||||
|
results = list(pool.map(lambda _: self.store.create('owner', path=str(self.model)), range(4)))
|
||||||
|
self.assertEqual(len({row['id'] for row in results}), 1)
|
||||||
|
self.assertEqual(len(self.store._records()), 1)
|
||||||
|
|
||||||
|
def test_invalid_legacy_directory_does_not_block_import(self):
|
||||||
|
first = self.store.create('owner', path=str(self.model))
|
||||||
|
config = self.model / 'config.json'
|
||||||
|
config.write_text('x' * len(config.read_text()))
|
||||||
|
replacement = self.root / 'replacement'
|
||||||
|
replacement.mkdir()
|
||||||
|
(replacement / 'config.json').write_text(json.dumps({'model_type': 'bert', 'transformers_version': '4.45'}))
|
||||||
|
(replacement / 'model.safetensors').write_bytes(b'test fixture, not runtime weights')
|
||||||
|
self.assertNotEqual(self.store.create('owner', path=str(replacement))['id'], first['id'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':unittest.main()
|
||||||
|
|
@ -0,0 +1,67 @@
|
||||||
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from django.test import TestCase
|
||||||
|
from rest_framework.exceptions import ValidationError
|
||||||
|
|
||||||
|
from common.constants.permission_constants import RoleConstants
|
||||||
|
from common.exception.app_exception import AppUnauthorizedFailed
|
||||||
|
from models_provider.models import Model
|
||||||
|
from models_provider.views.runtime import ModelLoading
|
||||||
|
|
||||||
|
|
||||||
|
class LoadingApiTests(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.model = Model.objects.create(name='loading-test', provider='model_local_provider',
|
||||||
|
model_type='EMBEDDING', model_name='test-only', credential='test-only', meta={'existing': 'keep'})
|
||||||
|
self.admin = SimpleNamespace(auth=SimpleNamespace(role_list=[str(RoleConstants.ADMIN.value)], permission_list=[]), data={})
|
||||||
|
self.view = ModelLoading()
|
||||||
|
|
||||||
|
def args(self):
|
||||||
|
return {'workspace_id': 'default', 'model_id': str(self.model.id)}
|
||||||
|
|
||||||
|
def test_policy_is_persistent_and_preserves_existing_metadata(self):
|
||||||
|
self.admin.data = {'strategy': 'PRELOAD', 'priority': 90}
|
||||||
|
with patch('models_provider.views.runtime.runtime_request', return_value={'status': 'QUEUED'}) as rpc:
|
||||||
|
response = self.view.put(self.admin, **self.args())
|
||||||
|
self.model.refresh_from_db()
|
||||||
|
self.assertEqual(self.model.meta['existing'], 'keep')
|
||||||
|
self.assertEqual(self.model.meta['loading_policy'], {'strategy': 'PRELOAD', 'priority': 90, 'idle_seconds': 300})
|
||||||
|
self.assertEqual(json.loads(response.content)['data']['status'], 'QUEUED')
|
||||||
|
self.assertEqual(rpc.call_args_list[0].args[0], 'configure')
|
||||||
|
|
||||||
|
def test_normal_user_cannot_change_policy_or_load_weights(self):
|
||||||
|
user = SimpleNamespace(auth=SimpleNamespace(role_list=[], permission_list=[]), data={'action': 'load'})
|
||||||
|
for method in (self.view.get, self.view.put, self.view.post):
|
||||||
|
with self.assertRaises(AppUnauthorizedFailed):
|
||||||
|
method(user, **self.args())
|
||||||
|
|
||||||
|
def test_workspace_and_unsupported_provider_are_rejected(self):
|
||||||
|
with self.assertRaises(ValidationError):
|
||||||
|
self.view.get(self.admin, workspace_id='other', model_id=str(self.model.id))
|
||||||
|
self.model.provider = 'aliyun_bai_lian_model_provider'
|
||||||
|
self.model.model_type = 'LLM'; self.model.save()
|
||||||
|
response = self.view.get(self.admin, **self.args())
|
||||||
|
self.assertEqual(json.loads(response.content)['data']['status'], 'EXTERNAL')
|
||||||
|
with self.assertRaises(ValidationError):
|
||||||
|
self.view.put(self.admin, **self.args())
|
||||||
|
|
||||||
|
def test_arbitrary_actions_and_invalid_policies_do_not_reach_runtime(self):
|
||||||
|
with patch('models_provider.views.runtime.runtime_request') as rpc:
|
||||||
|
self.admin.data = {'action': 'shell'}
|
||||||
|
with self.assertRaises(ValidationError): self.view.post(self.admin, **self.args())
|
||||||
|
self.admin.data = {'priority': 500}
|
||||||
|
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
|
||||||
|
rpc.assert_not_called()
|
||||||
|
self.model.refresh_from_db()
|
||||||
|
self.assertNotIn('loading_policy', self.model.meta)
|
||||||
|
|
||||||
|
def test_saved_policy_remains_available_when_runtime_is_temporarily_down(self):
|
||||||
|
self.admin.data = {'strategy': 'PRELOAD', 'priority': 80}
|
||||||
|
with patch('models_provider.views.runtime.runtime_request', side_effect=ValueError('offline')):
|
||||||
|
with self.assertRaises(ValidationError): self.view.put(self.admin, **self.args())
|
||||||
|
response = self.view.get(self.admin, **self.args())
|
||||||
|
self.model.refresh_from_db()
|
||||||
|
self.assertEqual(self.model.meta['loading_policy']['priority'], 80)
|
||||||
|
self.assertEqual(json.loads(response.content)['data']['status'], 'UNAVAILABLE')
|
||||||
|
|
@ -0,0 +1,199 @@
|
||||||
|
import contextlib
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
import weakref
|
||||||
|
import unittest
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from common.model_runtime import ModelRuntime, loading_policy
|
||||||
|
from common.model_runtime_client import signed_request, verify_request
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAdapter:
|
||||||
|
def __init__(self):
|
||||||
|
self.loaded, self.unloaded = [], []
|
||||||
|
self.native_ready = True
|
||||||
|
self.fail = False
|
||||||
|
self.block = None
|
||||||
|
|
||||||
|
def load(self, spec):
|
||||||
|
if self.block:
|
||||||
|
self.block.wait(3)
|
||||||
|
if self.fail:
|
||||||
|
raise ValueError('private error details')
|
||||||
|
self.loaded.append(spec['id'])
|
||||||
|
self.native_ready = True
|
||||||
|
return SimpleNamespace(), {'device': 'cpu'}
|
||||||
|
|
||||||
|
def unload(self, spec, value):
|
||||||
|
self.unloaded.append(spec['id'])
|
||||||
|
self.native_ready = False
|
||||||
|
|
||||||
|
def probe(self, spec):
|
||||||
|
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
|
||||||
|
|
||||||
|
def collect(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def error(self, exc):
|
||||||
|
return 'load failed'
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.now = 0
|
||||||
|
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
|
||||||
|
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
|
||||||
|
for key, priority in [('low', 10), ('high', 90)]}
|
||||||
|
self.adapter = FakeAdapter()
|
||||||
|
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
|
||||||
|
self.adapter, lambda: self.now)
|
||||||
|
|
||||||
|
def test_no_load_until_first_inference_and_then_reuses_weights(self):
|
||||||
|
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
||||||
|
self.assertEqual(self.adapter.loaded, [])
|
||||||
|
with self.manager.inference('low') as first:
|
||||||
|
self.assertIsNotNone(first)
|
||||||
|
with self.manager.inference('low') as second:
|
||||||
|
self.assertIs(first, second)
|
||||||
|
self.assertEqual(self.adapter.loaded, ['low'])
|
||||||
|
|
||||||
|
def test_startup_preloads_highest_priority_first(self):
|
||||||
|
self.specs['low']['policy']['strategy'] = 'PRELOAD'
|
||||||
|
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
||||||
|
self.manager.preload(['low', 'high'])
|
||||||
|
for key in ['low', 'high']:
|
||||||
|
self.manager.entries[key]['future'].result(3)
|
||||||
|
self.assertEqual(self.adapter.loaded, ['high', 'low'])
|
||||||
|
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
|
||||||
|
|
||||||
|
def test_concurrent_first_calls_only_load_once(self):
|
||||||
|
self.adapter.block = threading.Event()
|
||||||
|
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||||
|
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
|
||||||
|
self.adapter.block.set()
|
||||||
|
tokens = [future.result(3) for future in futures]
|
||||||
|
self.assertEqual(len(set(tokens)), 6)
|
||||||
|
self.assertEqual(self.adapter.loaded, ['low'])
|
||||||
|
self.assertEqual(self.manager.status('low')['active_requests'], 6)
|
||||||
|
for token in tokens:
|
||||||
|
self.manager.release('low', token)
|
||||||
|
|
||||||
|
def test_in_use_models_cannot_be_unloaded(self):
|
||||||
|
token = self.manager.acquire('low')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.manager.unload('low')
|
||||||
|
self.manager.release('low', token)
|
||||||
|
self.manager.unload('low')
|
||||||
|
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
|
||||||
|
|
||||||
|
def test_unload_releases_the_last_loader_reference(self):
|
||||||
|
class Weights:
|
||||||
|
pass
|
||||||
|
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
|
||||||
|
token = self.manager.acquire('low')
|
||||||
|
pointer = weakref.ref(self.manager.entries['low']['value'])
|
||||||
|
self.manager.release('low', token)
|
||||||
|
self.manager.unload('low')
|
||||||
|
self.assertIsNone(pointer())
|
||||||
|
|
||||||
|
def test_idle_cleanup_respects_active_and_preloaded_models(self):
|
||||||
|
active = self.manager.acquire('low')
|
||||||
|
self.specs['high']['policy']['strategy'] = 'PRELOAD'
|
||||||
|
self.manager.preload(['high'])
|
||||||
|
self.manager.entries['high']['future'].result(3)
|
||||||
|
self.now = 100
|
||||||
|
self.manager.sweep()
|
||||||
|
self.assertEqual(self.adapter.unloaded, [])
|
||||||
|
self.manager.release('low', active)
|
||||||
|
self.now = 131
|
||||||
|
self.manager.sweep()
|
||||||
|
self.assertEqual(self.adapter.unloaded, ['low'])
|
||||||
|
self.assertEqual(self.manager.status('high')['status'], 'READY')
|
||||||
|
|
||||||
|
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
|
||||||
|
self.adapter.fail = True
|
||||||
|
with self.assertRaisesRegex(ValueError, '^load failed$'):
|
||||||
|
self.manager.acquire('low')
|
||||||
|
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
|
||||||
|
self.adapter.fail = False
|
||||||
|
with self.manager.inference('low'):
|
||||||
|
pass
|
||||||
|
self.assertEqual(self.manager.status('low')['status'], 'READY')
|
||||||
|
|
||||||
|
def test_changed_model_config_never_reuses_old_weights(self):
|
||||||
|
token = self.manager.acquire('low')
|
||||||
|
self.specs['low']['fingerprint'] = 'v2'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.manager.acquire('low')
|
||||||
|
self.manager.release('low', token)
|
||||||
|
with self.manager.inference('low'):
|
||||||
|
pass
|
||||||
|
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
||||||
|
self.assertEqual(self.adapter.unloaded, ['low'])
|
||||||
|
|
||||||
|
def test_native_expiration_requires_loading_again(self):
|
||||||
|
self.specs['low']['driver'] = 'ollama'
|
||||||
|
token = self.manager.acquire('low')
|
||||||
|
self.manager.release('low', token)
|
||||||
|
self.adapter.native_ready = False
|
||||||
|
token = self.manager.acquire('low')
|
||||||
|
self.manager.release('low', token)
|
||||||
|
self.assertEqual(self.adapter.loaded, ['low', 'low'])
|
||||||
|
|
||||||
|
def test_policy_validation(self):
|
||||||
|
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
|
||||||
|
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
|
||||||
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||||
|
loading_policy(value)
|
||||||
|
|
||||||
|
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
|
||||||
|
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
|
||||||
|
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
|
||||||
|
self.assertEqual(verify_request(body, signature)['operation'], 'load')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
verify_request(body.replace(b'load', b'unload'), signature)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
verify_request(body, '')
|
||||||
|
with patch('common.model_runtime_client.time.time', return_value=1):
|
||||||
|
body, signature = signed_request({'operation': 'load'})
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
verify_request(body, signature)
|
||||||
|
|
||||||
|
|
||||||
|
class ProviderHookTests(unittest.TestCase):
|
||||||
|
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
|
||||||
|
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
|
||||||
|
events = []
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def lease(model_id):
|
||||||
|
events.append(('acquire', model_id))
|
||||||
|
try:
|
||||||
|
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
||||||
|
finally:
|
||||||
|
events.append(('release', model_id))
|
||||||
|
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
||||||
|
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
|
||||||
|
self.assertEqual(model.embed_query('text'), [1., 2.])
|
||||||
|
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
|
||||||
|
self.assertEqual(model._default_params['keep_alive'], -1)
|
||||||
|
|
||||||
|
def test_llm_stream_releases_lease_when_generator_is_closed(self):
|
||||||
|
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
|
||||||
|
from langchain_ollama import ChatOllama
|
||||||
|
events = []
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def lease(model_id):
|
||||||
|
events.append('acquire')
|
||||||
|
try:
|
||||||
|
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
|
||||||
|
finally:
|
||||||
|
events.append('release')
|
||||||
|
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
|
||||||
|
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
|
||||||
|
stream = model._create_chat_stream([])
|
||||||
|
next(stream); stream.close()
|
||||||
|
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
|
||||||
|
self.assertEqual(events, ['acquire', 'release'])
|
||||||
|
|
@ -131,11 +131,15 @@ def get_model_default_params(model):
|
||||||
return value
|
return value
|
||||||
return value
|
return value
|
||||||
|
|
||||||
return {
|
params = {
|
||||||
p.get('field'): convert_to_int(p.get('default_value'))
|
p.get('field'): convert_to_int(p.get('default_value'))
|
||||||
for p in model.model_params_form
|
for p in model.model_params_form
|
||||||
if p.get('default_value') is not None
|
if p.get('default_value') is not None
|
||||||
}
|
}
|
||||||
|
if model.model_type == 'LLM':
|
||||||
|
from models_provider.context_policy import model_context
|
||||||
|
params['context_tokens'] = model_context(params)
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs):
|
def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs):
|
||||||
|
|
@ -147,4 +151,8 @@ def get_model_instance_by_model_workspace_id(model_id, workspace_id, **kwargs):
|
||||||
"""
|
"""
|
||||||
model = get_model_by_id(model_id, workspace_id)
|
model = get_model_by_id(model_id, workspace_id)
|
||||||
s = get_model_default_params(model)
|
s = get_model_default_params(model)
|
||||||
return ModelManage.get_model(model_id, lambda _id: get_model(model, **{**s, **kwargs}))
|
params = {**s, **kwargs}
|
||||||
|
if model.model_type == 'LLM' and any(kwargs.get(key) is not None for key in ('context_tokens', 'num_ctx')):
|
||||||
|
from models_provider.context_policy import model_context
|
||||||
|
params['context_tokens'] = model_context(kwargs)
|
||||||
|
return ModelManage.get_model(model_id, lambda _id: get_model(model, **params))
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue