From 6df950ec3fbb6c3f65c77f8420bb6b319b0fe71b Mon Sep 17 00:00:00 2001 From: panyy Date: Wed, 21 Oct 2026 15:48:54 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E4=BC=98=E5=8C=96=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/base_search_dataset_step.py | 4 +- .../impl/base_search_knowledge_node.py | 4 +- apps/common/event/listener_manage.py | 2 +- apps/common/forms/__init__.py | 1 + .../handle/impl/text/csv_split_handle.py | 20 +- .../handle/impl/text/doc_split_handle.py | 6 +- .../handle/impl/text/html_split_handle.py | 6 +- .../handle/impl/text/pdf_split_handle.py | 22 +- .../handle/impl/text/text_split_handle.py | 6 +- .../handle/impl/text/xls_split_handle.py | 26 +- .../handle/impl/text/xlsx_split_handle.py | 26 +- .../handle/impl/text/zip_split_handle.py | 8 +- .../commands/services/services/local_model.py | 3 +- apps/common/utils/split_model.py | 29 +- apps/knowledge/models/knowledge.py | 40 + apps/knowledge/serializers/common.py | 37 +- apps/knowledge/serializers/document.py | 41 +- apps/knowledge/serializers/knowledge.py | 22 +- apps/knowledge/serializers/paragraph.py | 3 +- apps/knowledge/task/embedding.py | 41 +- apps/knowledge/urls.py | 6 + apps/knowledge/vector/base_vector.py | 13 +- apps/knowledge/views/document.py | 2 +- apps/knowledge/views/knowledge.py | 151 +++ apps/local_model/apps.py | 6 + .../serializers/model_apply_serializers.py | 92 +- apps/local_model/urls.py | 2 + apps/local_model/views/model_apply.py | 5 +- apps/maxkb/conf.py | 5 +- apps/maxkb/settings/base/web.py | 1 + .../credential/llm.py | 14 + .../credential/embedding/model.py | 28 +- .../model/embedding/model.py | 17 +- .../model/embedding/web.py | 10 +- .../model/reranker/model.py | 8 +- .../model/reranker/web.py | 3 +- .../credential/embedding.py | 11 +- .../ollama_model_provider/model/embedding.py | 32 +- .../impl/ollama_model_provider/model/llm.py | 17 + apps/models_provider/urls.py | 4 + apps/ops/celery/__init__.py | 13 +- apps/system_manage/models/__init__.py | 1 + apps/system_manage/serializers/system.py | 5 +- .../serializers/user_resource_permission.py | 16 +- apps/system_manage/urls.py | 5 + apps/users/models/__init__.py | 1 + apps/users/serializers/login.py | 8 +- apps/users/serializers/user.py | 54 +- apps/users/tests.py | 417 ++++++++- apps/users/urls.py | 7 + apps/users/views/__init__.py | 2 + main.py | 2 +- ui/src/api/knowledge/knowledge.ts | 95 +- ui/src/api/system/user-manage.ts | 60 +- ui/src/api/type/user.ts | 21 +- ui/src/api/user/user.ts | 7 +- ui/src/components/app-table/index.vue | 22 +- ui/src/enums/model.ts | 9 + ui/src/layout/components/sidebar/index.vue | 13 + ui/src/router/index.ts | 2 +- ui/src/router/modules/system.ts | 15 + ui/src/router/routes.ts | 5 + ui/src/views/document/upload/SetRules.vue | 46 +- ui/src/views/knowledge/KnowledgeSetting.vue | 856 ++++++++++++------ ui/src/views/knowledge/component/BaseForm.vue | 21 +- ui/src/views/login/index.vue | 1 + ui/src/views/mindmap/index.vue | 42 +- .../model/component/CreateModelDialog.vue | 24 +- ui/src/views/model/component/EditModel.vue | 21 +- ui/src/views/model/component/ModelCard.vue | 13 + .../user-manage/component/UserDrawer.vue | 34 +- ui/src/views/system/user-manage/index.vue | 66 +- ui/src/views/tool/index.vue | 21 +- ui/vite.config.ts | 35 +- 74 files changed, 2131 insertions(+), 603 deletions(-) diff --git a/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py b/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py index 0073b53e3..6281ffccb 100644 --- a/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py +++ b/apps/application/chat_pipeline/step/search_dataset_step/impl/base_search_dataset_step.py @@ -1,3 +1,4 @@ +from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list # coding=utf-8 """ @project: maxkb @@ -66,8 +67,7 @@ class BaseSearchDatasetStep(ISearchDatasetStep): if model.model_type != "EMBEDDING": raise Exception(_("Model does not exist")) self.context['model_name'] = model.name - default_params = get_model_default_params(model) - embedding_model = ModelManage.get_model(model_id, lambda _id: get_model(model, **{**default_params})) + embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list) embedding_value = embedding_model.embed_query(exec_problem_text) vector = VectorStore.get_embedding_vector() embedding_list = vector.query(exec_problem_text, embedding_value, knowledge_id_list, None, exclude_document_id_list, diff --git a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py b/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py index 5621f0351..e7a0a676e 100644 --- a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py +++ b/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py @@ -1,3 +1,4 @@ +from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list # coding=utf-8 """ @project: maxkb @@ -102,7 +103,8 @@ class BaseSearchKnowledgeNode(ISearchKnowledgeStepNode): return get_none_result(question) model_id = get_embedding_id(knowledge_id_list) workspace_id = self.workflow_manage.get_body().get('workspace_id') - embedding_model = get_model_instance_by_model_workspace_id(model_id, workspace_id) + get_model_instance_by_model_workspace_id(model_id, workspace_id) # Preserve resource access validation. + embedding_model = get_embedding_model_by_knowledge_id_list(knowledge_id_list) embedding_value = embedding_model.embed_query(question) vector = VectorStore.get_embedding_vector() exclude_document_id_list = [str(document.id) for document in diff --git a/apps/common/event/listener_manage.py b/apps/common/event/listener_manage.py index 318b2f0bb..1bdc0e65b 100644 --- a/apps/common/event/listener_manage.py +++ b/apps/common/event/listener_manage.py @@ -357,7 +357,7 @@ class ListenerManagement: @staticmethod def update_problem(args: UpdateProblemArgs): problem_paragraph_mapping_list = QuerySet(ProblemParagraphMapping).filter(problem_id=args.problem_id) - embed_value = args.embedding_model.embed_query(args.problem_content) + embed_value = args.embedding_model.embed_documents([args.problem_content])[0] VectorStore.get_embedding_vector().update_by_source_ids([v.id for v in problem_paragraph_mapping_list], {'embedding': embed_value}) diff --git a/apps/common/forms/__init__.py b/apps/common/forms/__init__.py index 609542193..251f01df0 100644 --- a/apps/common/forms/__init__.py +++ b/apps/common/forms/__init__.py @@ -22,3 +22,4 @@ from .table_checkbox import * from .radio_card_field import * from .label import * from .slider_field import * +from .switch_field import * diff --git a/apps/common/handle/impl/text/csv_split_handle.py b/apps/common/handle/impl/text/csv_split_handle.py index 569a16881..e10f7a93d 100644 --- a/apps/common/handle/impl/text/csv_split_handle.py +++ b/apps/common/handle/impl/text/csv_split_handle.py @@ -16,6 +16,7 @@ from charset_normalizer import detect from common.handle.base_split_handle import BaseSplitHandle from common.utils.logger import maxkb_logger +from common.utils.table_split import split_table_rows def post_cell(cell_value): @@ -28,7 +29,7 @@ def row_to_md(row): class CsvSplitHandle(BaseSplitHandle): - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): buffer = get_buffer(file) paragraphs = [] file_name = os.path.basename(file.name) @@ -46,22 +47,7 @@ class CsvSplitHandle(BaseSplitHandle): return result if len(title_row_list) == 0: return result - result_item_content = '' - for row in reader: - next_md_content = row_to_md(row) - next_md_content_len = len(next_md_content) - result_item_content_len = len(result_item_content) - if len(result_item_content) == 0: - result_item_content += title_md_content - result_item_content += next_md_content - else: - if result_item_content_len + next_md_content_len < limit: - result_item_content += next_md_content - else: - paragraphs.append({'content': result_item_content, 'title': ''}) - result_item_content = title_md_content + next_md_content - if len(result_item_content) > 0: - paragraphs.append({'content': result_item_content, 'title': ''}) + paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in reader), limit, overlap)) return result except Exception as e: maxkb_logger.error(f"Error processing CSV file {file.name}: {e}, {traceback.format_exc()}") diff --git a/apps/common/handle/impl/text/doc_split_handle.py b/apps/common/handle/impl/text/doc_split_handle.py index 33f234767..27abc19c0 100644 --- a/apps/common/handle/impl/text/doc_split_handle.py +++ b/apps/common/handle/impl/text/doc_split_handle.py @@ -192,7 +192,7 @@ class DocSplitHandle(BaseSplitHandle): for element in elements]) - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): file_name = os.path.basename(file.name) try: if type(limit) is str: @@ -206,9 +206,9 @@ class DocSplitHandle(BaseSplitHandle): if len(image_list) > 0: save_image(image_list) if pattern_list is not None and len(pattern_list) > 0: - split_model = SplitModel(pattern_list, with_filter, limit) + split_model = SplitModel(pattern_list, with_filter, limit, overlap) else: - split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) + split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap) except BaseException as e: maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}") return { diff --git a/apps/common/handle/impl/text/html_split_handle.py b/apps/common/handle/impl/text/html_split_handle.py index a82cfdaec..9175b4e02 100644 --- a/apps/common/handle/impl/text/html_split_handle.py +++ b/apps/common/handle/impl/text/html_split_handle.py @@ -44,16 +44,16 @@ class HTMLSplitHandle(BaseSplitHandle): return True return False - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): buffer = get_buffer(file) if type(limit) is str: limit = int(limit) if type(with_filter) is str: with_filter = with_filter.lower() == 'true' if pattern_list is not None and len(pattern_list) > 0: - split_model = SplitModel(pattern_list, with_filter, limit) + split_model = SplitModel(pattern_list, with_filter, limit, overlap) else: - split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) + split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap) try: encoding = get_encoding(buffer) content = buffer.decode(encoding) diff --git a/apps/common/handle/impl/text/pdf_split_handle.py b/apps/common/handle/impl/text/pdf_split_handle.py index eca98b0f0..4e22031f1 100644 --- a/apps/common/handle/impl/text/pdf_split_handle.py +++ b/apps/common/handle/impl/text/pdf_split_handle.py @@ -42,7 +42,7 @@ def check_links_in_pdf(doc): class PdfSplitHandle(BaseSplitHandle): - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): with tempfile.NamedTemporaryFile(delete=False) as temp_file: # 将上传的文件保存到临时文件中 for chunk in file.chunks(): @@ -57,12 +57,12 @@ class PdfSplitHandle(BaseSplitHandle): if type(with_filter) is str: with_filter = with_filter.lower() == 'true' # 处理有目录的pdf - result = self.handle_toc(pdf_document, limit) + result = self.handle_toc(pdf_document, limit, overlap) if result is not None: return {'name': file.name, 'content': result} # 没目录但是有链接的pdf - result = self.handle_links(pdf_document, pattern_list, with_filter, limit) + result = self.handle_links(pdf_document, pattern_list, with_filter, limit, overlap) if result is not None and len(result) > 0: return {'name': file.name, 'content': result} @@ -70,9 +70,9 @@ class PdfSplitHandle(BaseSplitHandle): content = self.handle_pdf_content(file, pdf_document) if pattern_list is not None and len(pattern_list) > 0: - split_model = SplitModel(pattern_list, with_filter, limit) + split_model = SplitModel(pattern_list, with_filter, limit, overlap) else: - split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) + split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap) except BaseException as e: maxkb_logger.error(f"File: {file.name}, error: {e}, {traceback.format_exc()}") return { @@ -131,7 +131,7 @@ class PdfSplitHandle(BaseSplitHandle): return content @staticmethod - def handle_toc(doc, limit): + def handle_toc(doc, limit, overlap: int = 0): # 找到目录 toc = doc.get_toc() if toc is None or len(toc) == 0: @@ -183,7 +183,7 @@ class PdfSplitHandle(BaseSplitHandle): real_chapter_title = chapter_title[:256] # 限制章节内容长度 if 0 < limit < len(chapter_text): - split_text = smart_split_paragraph(chapter_text, limit) + split_text = smart_split_paragraph(chapter_text, limit, overlap) for text in split_text: chapters.append({"title": real_chapter_title, "content": text}) else: @@ -193,7 +193,7 @@ class PdfSplitHandle(BaseSplitHandle): return chapters @staticmethod - def handle_links(doc, pattern_list, with_filter, limit): + def handle_links(doc, pattern_list, with_filter, limit, overlap: int = 0): # 检查文档是否包含内部链接 if not check_links_in_pdf(doc): return @@ -262,7 +262,7 @@ class PdfSplitHandle(BaseSplitHandle): # 限制章节内容长度 if 0 < limit < len(chapter_text): - split_text = smart_split_paragraph(chapter_text, limit) + split_text = smart_split_paragraph(chapter_text, limit, overlap) for text in split_text: chapters.append({"title": link_title, "content": text}) else: @@ -285,9 +285,9 @@ class PdfSplitHandle(BaseSplitHandle): except BaseException as e: maxkb_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e)) if pattern_list is not None and len(pattern_list) > 0: - split_model = SplitModel(pattern_list, with_filter, limit) + split_model = SplitModel(pattern_list, with_filter, limit, overlap) else: - split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) + split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap) # 插入目录前的部分 page_content = re.sub(r'(? 0: - split_model = SplitModel(pattern_list, with_filter, limit) + split_model = SplitModel(pattern_list, with_filter, limit, overlap) else: - split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) + split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit, overlap=overlap) try: content = buffer.decode(detect(buffer)['encoding']) except BaseException as e: diff --git a/apps/common/handle/impl/text/xls_split_handle.py b/apps/common/handle/impl/text/xls_split_handle.py index 3b4bb0918..cb7d77d01 100644 --- a/apps/common/handle/impl/text/xls_split_handle.py +++ b/apps/common/handle/impl/text/xls_split_handle.py @@ -13,6 +13,7 @@ import xlrd from common.handle.base_split_handle import BaseSplitHandle from common.utils.logger import maxkb_logger +from common.utils.table_split import split_table_rows def post_cell(cell_value): @@ -24,7 +25,7 @@ def row_to_md(row): [post_cell(str(cell)) if cell is not None else '' for cell in row]) + ' |\n' -def handle_sheet(file_name, sheet, limit: int): +def handle_sheet(file_name, sheet, limit: int, overlap: int = 0): rows = iter([sheet.row_values(i) for i in range(sheet.nrows)]) paragraphs = [] result = {'name': file_name, 'content': paragraphs} @@ -37,27 +38,12 @@ def handle_sheet(file_name, sheet, limit: int): return result if len(title_row_list) == 0: return result - result_item_content = '' - for row in rows: - next_md_content = row_to_md(row) - next_md_content_len = len(next_md_content) - result_item_content_len = len(result_item_content) - if len(result_item_content) == 0: - result_item_content += title_md_content - result_item_content += next_md_content - else: - if result_item_content_len + next_md_content_len < limit: - result_item_content += next_md_content - else: - paragraphs.append({'content': result_item_content, 'title': ''}) - result_item_content = title_md_content + next_md_content - if len(result_item_content) > 0: - paragraphs.append({'content': result_item_content, 'title': ''}) + paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row) for row in rows), limit, overlap)) return result class XlsSplitHandle(BaseSplitHandle): - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): buffer = get_buffer(file) try: if type(limit) is str: @@ -67,8 +53,8 @@ class XlsSplitHandle(BaseSplitHandle): worksheets_size = len(worksheets) return [row for row in [handle_sheet(file.name, - sheet, limit) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet( - sheet.name, sheet, limit) for sheet + sheet, limit, overlap) if worksheets_size == 1 and sheet.name == 'Sheet1' else handle_sheet( + sheet.name, sheet, limit, overlap) for sheet in worksheets] if row is not None] except Exception as e: maxkb_logger.error(f"Error processing XLS file {file.name}: {e}, {traceback.format_exc()}") diff --git a/apps/common/handle/impl/text/xlsx_split_handle.py b/apps/common/handle/impl/text/xlsx_split_handle.py index 55f3786ce..fd467363d 100644 --- a/apps/common/handle/impl/text/xlsx_split_handle.py +++ b/apps/common/handle/impl/text/xlsx_split_handle.py @@ -15,6 +15,7 @@ import openpyxl from common.handle.base_split_handle import BaseSplitHandle from common.handle.impl.common_handle import xlsx_embed_cells_images from common.utils.logger import maxkb_logger +from common.utils.table_split import split_table_rows def post_cell(image_dict, cell_value): @@ -30,7 +31,7 @@ def row_to_md(row, image_dict): in row]) + ' |\n' -def handle_sheet(file_name, sheet, image_dict, limit: int): +def handle_sheet(file_name, sheet, image_dict, limit: int, overlap: int = 0): rows = sheet.rows paragraphs = [] result = {'name': file_name, 'content': paragraphs} @@ -43,27 +44,12 @@ def handle_sheet(file_name, sheet, image_dict, limit: int): return result if len(title_row_list) == 0: return result - result_item_content = '' - for row in rows: - next_md_content = row_to_md(row, image_dict) - next_md_content_len = len(next_md_content) - result_item_content_len = len(result_item_content) - if len(result_item_content) == 0: - result_item_content += title_md_content - result_item_content += next_md_content - else: - if result_item_content_len + next_md_content_len < limit: - result_item_content += next_md_content - else: - paragraphs.append({'content': result_item_content, 'title': ''}) - result_item_content = title_md_content + next_md_content - if len(result_item_content) > 0: - paragraphs.append({'content': result_item_content, 'title': ''}) + paragraphs.extend(split_table_rows(title_md_content, (row_to_md(row, image_dict) for row in rows), limit, overlap)) return result class XlsxSplitHandle(BaseSplitHandle): - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): buffer = get_buffer(file) try: if type(limit) is str: @@ -80,8 +66,8 @@ class XlsxSplitHandle(BaseSplitHandle): [handle_sheet(file.name, sheet, image_dict, - limit) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet( - sheet.title, sheet, image_dict, limit) for sheet + limit, overlap) if worksheets_size == 1 and sheet.title == 'Sheet1' else handle_sheet( + sheet.title, sheet, image_dict, limit, overlap) for sheet in worksheets] if row is not None] except Exception as e: maxkb_logger.error(f"Error processing XLSX file {file.name}: {e}, {traceback.format_exc()}") diff --git a/apps/common/handle/impl/text/zip_split_handle.py b/apps/common/handle/impl/text/zip_split_handle.py index 6609a981c..93110a97f 100644 --- a/apps/common/handle/impl/text/zip_split_handle.py +++ b/apps/common/handle/impl/text/zip_split_handle.py @@ -50,11 +50,11 @@ split_handles = [ ] -def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image): +def file_to_paragraph(file, pattern_list: List, with_filter: bool, limit: int, save_inner_image, overlap: int = 0): get_buffer = FileBufferHandle().get_buffer for split_handle in split_handles: if split_handle.support(file, get_buffer): - return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image) + return split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, save_inner_image, overlap) raise Exception(_('Unsupported file format')) @@ -116,7 +116,7 @@ def filter_image_file(result_list: list, image_list): class ZipSplitHandle(BaseSplitHandle): - def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image): + def handle(self, file, pattern_list: List, with_filter: bool, limit: int, get_buffer, save_image, overlap: int = 0): if type(limit) is str: limit = int(limit) if type(with_filter) is str: @@ -137,7 +137,7 @@ class ZipSplitHandle(BaseSplitHandle): try: # 处理一下文件名 f.name = get_file_name(f.name) - value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image) + value = file_to_paragraph(f, pattern_list, with_filter, limit, save_image, overlap) if isinstance(value, list): result = [*result, *value] else: diff --git a/apps/common/management/commands/services/services/local_model.py b/apps/common/management/commands/services/services/local_model.py index a37f0d16b..3f485d487 100644 --- a/apps/common/management/commands/services/services/local_model.py +++ b/apps/common/management/commands/services/services/local_model.py @@ -26,7 +26,8 @@ class GunicornLocalModelService(BaseService): print("\n- Start Gunicorn Local Model WSGI HTTP Server") log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s ' bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}' - worker = CONFIG.get("LOCAL_MODEL_HOST_WORKER", 1) + # Weights and the load queue have one owner; inference remains multithreaded. + worker = 1 max_requests = 10240 if int(worker) > 1 else 0 cmd = [ 'gunicorn', 'maxkb.wsgi:application', diff --git a/apps/common/utils/split_model.py b/apps/common/utils/split_model.py index 8bf2df59e..0fdd30717 100644 --- a/apps/common/utils/split_model.py +++ b/apps/common/utils/split_model.py @@ -274,16 +274,22 @@ def post_handler_paragraph(content: str, limit: int): return reduce(lambda x, y: [*x, *y], map(lambda row: re.findall(pattern, row), result), []) -def smart_split_paragraph(content: str, limit: int): +def smart_split_paragraph(content: str, limit: int, overlap: int = 0): """ 智能分段:在limit前找到合适的分割点(句号、回车等) :param content: 需要分段的文本 :param limit: 最大字符限制 + :param overlap: 相邻分段的重叠长度 :return: 分段后的文本列表 """ if len(content) <= limit: return [content] + if overlap is None or overlap < 0: + overlap = 0 + if overlap >= limit: + overlap = max(0, limit // 2) + result = [] start = 0 @@ -318,7 +324,9 @@ def smart_split_paragraph(content: str, limit: int): best_split = end result.append(content[start:best_split]) - start = best_split + # 相邻分段保留 overlap 字符重叠,且保证向前推进 + next_start = best_split - overlap if overlap > 0 else best_split + start = next_start if next_start > start else best_split return [text for text in result if text.strip()] @@ -345,7 +353,7 @@ def filter_special_char(content: str): class SplitModel: - def __init__(self, content_level_pattern, with_filter=True, limit=100000): + def __init__(self, content_level_pattern, with_filter=True, limit=100000, overlap=0): self.content_level_pattern = content_level_pattern self.with_filter = with_filter if type(limit) is not int: @@ -355,6 +363,16 @@ class SplitModel: if limit < 50: limit = 50 self.limit = limit + if type(overlap) is not int: + try: + overlap = int(overlap) + except (TypeError, ValueError): + overlap = 0 + if overlap is None or overlap < 0: + overlap = 0 + if overlap >= self.limit: + overlap = max(0, self.limit // 2) + self.overlap = overlap def parse_to_tree(self, text: str, index=0): """ @@ -365,7 +383,8 @@ class SplitModel: """ level_content_list = parse_title_level(text, self.content_level_pattern, index) if len(level_content_list) == 0: - return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit)] + return [to_tree_obj(row, 'block') for row in smart_split_paragraph(text, limit=self.limit, + overlap=self.overlap)] if index == 0 and text.lstrip().index(level_content_list[0]["content"].lstrip()) != 0: level_content_list.insert(0, to_tree_obj("")) @@ -374,7 +393,7 @@ class SplitModel: for i in range(len(level_title_content_list)): start_content: str = level_title_content_list[i].get('content') if cursor < text.index(start_content, cursor): - for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit): + for row in smart_split_paragraph(text[cursor: text.index(start_content, cursor)], limit=self.limit, overlap=self.overlap): level_content_list.insert(0, to_tree_obj(row, 'block')) block, cursor = get_level_block(text, level_title_content_list, i, cursor) diff --git a/apps/knowledge/models/knowledge.py b/apps/knowledge/models/knowledge.py index dca940b48..3b1a43637 100644 --- a/apps/knowledge/models/knowledge.py +++ b/apps/knowledge/models/knowledge.py @@ -56,6 +56,46 @@ class KnowledgeScope(models.TextChoices): WORKSPACE = "WORKSPACE", "工作空间可用" +class RebuildStatus(models.TextChoices): + PENDING = "PENDING", "待执行" + RUNNING = "RUNNING", "执行中" + SUCCESS = "SUCCESS", "成功" + PARTIAL = "PARTIAL", "部分成功" + FAILED = "FAILED", "失败" + + +class KnowledgeRebuildTask(AppModelMixin): + """知识库独立索引重建任务。 + + 与“更换向量模型自动触发重向量化”分开:本表记录管理员在配置不变时 + 主动发起的重建请求、执行状态、文档统计和失败原因。 + """ + + id = models.UUIDField(primary_key=True, max_length=128, default=uuid.uuid7, editable=False, verbose_name="主键id") + knowledge = models.ForeignKey('knowledge.Knowledge', on_delete=models.CASCADE, db_constraint=False, + related_name="rebuild_tasks", verbose_name="知识库") + workspace_id = models.CharField(max_length=64, verbose_name="工作空间id", default="default", db_index=True) + status = models.CharField(max_length=20, choices=RebuildStatus.choices, default=RebuildStatus.PENDING, + db_index=True, verbose_name="任务状态") + config_snapshot = models.JSONField(verbose_name="配置快照", default=dict) + document_total = models.IntegerField(verbose_name="文档总数", default=0) + document_success = models.IntegerField(verbose_name="成功文档数", default=0) + document_failed = models.IntegerField(verbose_name="失败文档数", default=0) + error_message = models.TextField(verbose_name="失败原因", default="", blank=True) + operator = models.ForeignKey(User, on_delete=models.SET_NULL, db_constraint=False, null=True, blank=True, + related_name="+", verbose_name="操作人") + started_at = models.DateTimeField(verbose_name="开始时间", null=True, blank=True) + finished_at = models.DateTimeField(verbose_name="结束时间", null=True, blank=True) + celery_task_id = models.CharField(max_length=128, verbose_name="Celery任务id", default="", blank=True) + attempt = models.IntegerField(verbose_name="汇总轮询次数", default=0) + + class Meta: + db_table = "knowledge_rebuild_task" + indexes = [ + models.Index(fields=["knowledge", "status"]), + ] + + class HitHandlingMethod(models.TextChoices): optimization = 'optimization', '模型优化' directly_return = 'directly_return', '直接返回' diff --git a/apps/knowledge/serializers/common.py b/apps/knowledge/serializers/common.py index be796ea4a..87b5a89bf 100644 --- a/apps/knowledge/serializers/common.py +++ b/apps/knowledge/serializers/common.py @@ -114,40 +114,31 @@ class ProblemParagraphManage: def get_embedding_model_by_knowledge_id_list(knowledge_id_list: List): - knowledge_list = QuerySet(Knowledge).filter(id__in=knowledge_id_list) - if len(set([knowledge.embedding_model_id for knowledge in knowledge_list])) > 1: - raise Exception(_('The knowledge base is inconsistent with the vector model')) - if len(knowledge_list) == 0: + knowledge_list = list(QuerySet(Knowledge).select_related('embedding_model').filter(id__in=knowledge_id_list)) + if not knowledge_list: raise Exception(_('Knowledge base setting error, please reset the knowledge base')) - if knowledge_list[0].embedding_model_id is None or knowledge_list[0].embedding_model is None: - raise Exception(_('Knowledge base setting error, please reset the knowledge base')) - - default_params = get_model_default_params(knowledge_list[0].embedding_model) - - return ModelManage.get_model( - str(knowledge_list[0].embedding_model_id), - lambda _id: get_model(knowledge_list[0].embedding_model, **{**default_params}) - ) + from knowledge.embedding_params import knowledge_params + configs = [(str(k.embedding_model_id), {key: value for key, value in knowledge_params(k).items() + if key != 'batch_size'}) for k in knowledge_list] + if any(config != configs[0] for config in configs[1:]): + raise Exception('知识库的向量模型或嵌入参数不一致') + return get_embedding_model_by_knowledge(knowledge_list[0]) def get_embedding_model_by_knowledge_id(knowledge_id: str): knowledge = QuerySet(Knowledge).select_related('embedding_model').filter(id=knowledge_id).first() - if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None: - raise Exception(_('Knowledge base setting error, please reset the knowledge base')) - - default_params = get_model_default_params(knowledge.embedding_model) - - return ModelManage.get_model(str(knowledge.embedding_model_id), - lambda _id: get_model(knowledge.embedding_model, **{**default_params})) + return get_embedding_model_by_knowledge(knowledge) def get_embedding_model_by_knowledge(knowledge): if knowledge is None or knowledge.embedding_model_id is None or knowledge.embedding_model is None: raise Exception(_('Knowledge base setting error, please reset the knowledge base')) + from knowledge.embedding_params import ConfiguredEmbedding, knowledge_params default_params = get_model_default_params(knowledge.embedding_model) - - return ModelManage.get_model(str(knowledge.embedding_model_id), - lambda _id: get_model(knowledge.embedding_model, **{**default_params})) + model = ModelManage.get_model(str(knowledge.embedding_model_id), + lambda _id: get_model(knowledge.embedding_model, **default_params)) + params = knowledge_params(knowledge) + return ConfiguredEmbedding(model, params) if params else model def get_embedding_model_id_by_knowledge_id(knowledge_id): diff --git a/apps/knowledge/serializers/document.py b/apps/knowledge/serializers/document.py index 16ab61773..0df156081 100644 --- a/apps/knowledge/serializers/document.py +++ b/apps/knowledge/serializers/document.py @@ -162,7 +162,8 @@ class DocumentEditInstanceSerializer(serializers.Serializer): class DocumentSplitRequest(serializers.Serializer): file = serializers.ListField(required=True, label=_('file list')) - limit = serializers.IntegerField(required=False, label=_('limit')) + limit = serializers.IntegerField(required=False, min_value=50, max_value=100000, label=_('limit')) + overlap = serializers.IntegerField(required=False, min_value=0, label=_('chunk overlap')) patterns = serializers.ListField( required=False, child=serializers.CharField(required=True, label=_('patterns')), @@ -338,7 +339,8 @@ class DocumentSerializers(serializers.Serializer): else: document_list.update(knowledge_id=target_knowledge_id) model_id = None - if knowledge.embedding_model_id != target_knowledge.embedding_model_id: + if (knowledge.embedding_model_id != target_knowledge.embedding_model_id or + (knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})): model_id = get_embedding_model_id_by_knowledge_id(target_knowledge_id) pid_list = [paragraph.id for paragraph in paragraph_list] @@ -1047,7 +1049,21 @@ class DocumentSerializers(serializers.Serializer): def parse(self, instance): self.is_valid(instance=instance, raise_exception=True) - DocumentSplitRequest(data=instance).is_valid(raise_exception=True) + split_request = DocumentSplitRequest(data=instance) + split_request.is_valid(raise_exception=True) + instance = split_request.validated_data + + # 分段长度/重叠未显式指定时,回退到知识库分段规则(meta.segmentation) + knowledge = QuerySet(Knowledge).filter(id=self.data.get('knowledge_id')).first() + segmentation = (knowledge.meta or {}).get('segmentation') if knowledge is not None else None + segmentation = segmentation or {} + default_limit = segmentation.get('chunk_size') or 4096 + default_overlap = segmentation.get('chunk_overlap') or 0 + limit = instance.get("limit", None) or default_limit + overlap = instance.get("overlap", None) + overlap = default_overlap if overlap is None else overlap + if not 0 <= overlap < limit: + raise AppApiException(500, "重叠长度必须满足 0 ≤ 重叠长度 < 分段长度") file_list = instance.get("file") return reduce( @@ -1056,7 +1072,8 @@ class DocumentSerializers(serializers.Serializer): f, instance.get("patterns", None), instance.get("with_filter", None), - instance.get("limit", 4096) + limit, + overlap ) for f in file_list], [] ) @@ -1095,7 +1112,7 @@ class DocumentSerializers(serializers.Serializer): item['source_file_id'] = file_id return result_list - def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int): + def file_to_paragraph(self, file, pattern_list: List, with_filter: bool, limit: int, overlap: int = 0): # 保存源文件 file_id = uuid.uuid7() raw_file = File( @@ -1108,12 +1125,22 @@ class DocumentSerializers(serializers.Serializer): raw_file.save(file.read()) file.seek(0) + from system_manage.ai_hub_knowledge import split_with_parser + parsed = split_with_parser(self.data.get('workspace_id'), self.data.get('knowledge_id'), + self.context.get('user'), file, pattern_list, with_filter, limit, overlap) + if parsed is not None: + result, provenance = parsed + raw_file.meta = {**(raw_file.meta or {}), 'parser': provenance} + QuerySet(File).filter(id=file_id).update(meta=raw_file.meta) + return self.normalize_split_result(result, file, default_split_handle, file_id) get_buffer = FileBufferHandle().get_buffer for split_handle in split_handles: if split_handle.support(file, get_buffer): - result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image) + result = split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image, + overlap) return self.normalize_split_result(result, file, split_handle, file_id) - result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image) + result = default_split_handle.handle(file, pattern_list, with_filter, limit, get_buffer, self.save_image, + overlap) return self.normalize_split_result(result, file, default_split_handle, file_id) class SplitPattern(serializers.Serializer): diff --git a/apps/knowledge/serializers/knowledge.py b/apps/knowledge/serializers/knowledge.py index 8a80cfa25..6725d2d6f 100644 --- a/apps/knowledge/serializers/knowledge.py +++ b/apps/knowledge/serializers/knowledge.py @@ -70,6 +70,7 @@ class KnowledgeWebCreateRequest(serializers.Serializer): class KnowledgeEditRequest(serializers.Serializer): + embedding_params = serializers.DictField(required=False) name = serializers.CharField(required=False, max_length=64, min_length=1, label=_('knowledge name')) desc = serializers.CharField(required=False, max_length=256, min_length=1, label=_('knowledge description')) meta = serializers.DictField(required=False) @@ -361,7 +362,7 @@ class KnowledgeSerializer(serializers.Serializer): @transaction.atomic def edit(self, instance: Dict, select_one=True): self.is_valid() - knowledge = QuerySet(Knowledge).get(id=self.data.get("knowledge_id")) + knowledge = QuerySet(Knowledge).select_for_update().get(id=self.data.get("knowledge_id")) if QuerySet(Knowledge).filter( workspace_id=knowledge.workspace_id, name=instance.get('name'), @@ -376,7 +377,20 @@ class KnowledgeSerializer(serializers.Serializer): if 'desc' in instance: knowledge.desc = instance.get("desc") if 'meta' in instance: - knowledge.meta = instance.get('meta') + incoming_meta = dict(instance.get('meta') or {}) + # Independently saved metadata/rules must survive stale general settings forms. + for managed_key in ('business_metadata', 'segmentation', 'parser_id', 'embedding_params'): + incoming_meta.pop(managed_key, None) + if managed_key in (knowledge.meta or {}): + incoming_meta[managed_key] = knowledge.meta[managed_key] + knowledge.meta = incoming_meta + if 'embedding_params' in instance: + from knowledge.embedding_params import validate_params + try: + params = validate_params(instance['embedding_params']) + except ValueError as exc: + raise AppApiException(500, str(exc)) + knowledge.meta = {**(knowledge.meta or {}), 'embedding_params': params} if 'folder_id' in instance: knowledge.folder_id = instance.get('folder_id') if 'file_size_limit' in instance: @@ -384,6 +398,10 @@ class KnowledgeSerializer(serializers.Serializer): if 'file_count_limit' in instance: knowledge.file_count_limit = instance.get('file_count_limit') if 'user_id' in instance: + if str(instance.get('user_id')) != str(knowledge.user_id): + from knowledge.serializers.knowledge_ext import _creator_candidates + if not _creator_candidates(knowledge.workspace_id).filter(id=instance.get('user_id')).exists(): + raise AppApiException(500, '创建者须为已激活且启用的成员') knowledge.user_id = instance.get('user_id') if 'create_user' in instance: knowledge.create_user = instance.get('create_user') diff --git a/apps/knowledge/serializers/paragraph.py b/apps/knowledge/serializers/paragraph.py index a1df1bd81..133e19dd4 100644 --- a/apps/knowledge/serializers/paragraph.py +++ b/apps/knowledge/serializers/paragraph.py @@ -597,7 +597,8 @@ class ParagraphSerializers(serializers.Serializer): target_knowledge = QuerySet(Knowledge).filter(id=target_knowledge_id).first() knowledge = QuerySet(Knowledge).filter(id=knowledge_id).first() embedding_model_id = None - if target_knowledge.embedding_model_id != knowledge.embedding_model_id: + if (target_knowledge.embedding_model_id != knowledge.embedding_model_id or + (knowledge.meta or {}).get('embedding_params', {}) != (target_knowledge.meta or {}).get('embedding_params', {})): embedding_model_id = str(target_knowledge.embedding_model_id) pid_list = [paragraph.id for paragraph in paragraph_list] # 修改段落信息 diff --git a/apps/knowledge/task/embedding.py b/apps/knowledge/task/embedding.py index 750c28e15..11fbc52f4 100644 --- a/apps/knowledge/task/embedding.py +++ b/apps/knowledge/task/embedding.py @@ -11,7 +11,7 @@ from common.config.embedding_config import ModelManage from common.event.listener_manage import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \ UpdateEmbeddingDocumentIdArgs from common.utils.logger import maxkb_logger -from knowledge.models import Document, TaskType, State +from knowledge.models import Document, Paragraph, Problem, TaskType, State from knowledge.serializers.common import drop_knowledge_index from models_provider.models import Model from models_provider.tools import get_model, get_model_default_params @@ -35,21 +35,35 @@ def get_embedding_model(model_id, exception_handler=lambda e: maxkb_logger.error return embedding_model +def scoped_model(model_id, knowledge_id): + from knowledge.serializers.common import get_embedding_model_by_knowledge_id + if knowledge_id is None: + raise ValueError('无法确定向量化任务的知识库') + return get_embedding_model_by_knowledge_id(knowledge_id) + + +def paragraph_knowledge(ids): + values = list(QuerySet(Paragraph).filter(id__in=ids).values_list('knowledge_id', flat=True).distinct()) + if len(values) != 1: + raise ValueError('段落须属于同一知识库') + return values[0] + + @celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id']}, name='celery:embedding_by_paragraph') def embedding_by_paragraph(paragraph_id, model_id): - embedding_model = get_embedding_model(model_id) + embedding_model = scoped_model(model_id, paragraph_knowledge([paragraph_id])) ListenerManagement.embedding_by_paragraph(paragraph_id, embedding_model) @celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_data_list') def embedding_by_paragraph_data_list(data_list, paragraph_id_list, model_id): - embedding_model = get_embedding_model(model_id) + embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list)) ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model) @celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, name='celery:embedding_by_paragraph_list') def embedding_by_paragraph_list(paragraph_id_list, model_id): - embedding_model = get_embedding_model(model_id) + embedding_model = scoped_model(model_id, paragraph_knowledge(paragraph_id_list)) ListenerManagement.embedding_by_paragraph_list(paragraph_id_list, embedding_model) @@ -77,7 +91,11 @@ def embedding_by_document(document_id, model_id, state_list=None): traceback=traceback.format_exc() )) - embedding_model = get_embedding_model(model_id, exception_handler) + try: + embedding_model = scoped_model(model_id, QuerySet(Document).filter(id=document_id).values_list('knowledge_id', flat=True).first()) + except Exception as exc: + exception_handler(exc) + raise # ListenerManagement.embedding_by_document(document_id, embedding_model, state_list) @@ -130,12 +148,15 @@ def embedding_by_problem(args, model_id): @param model_id: 模型id @return: """ - embedding_model = get_embedding_model(model_id) + embedding_model = scoped_model(model_id, args.get('knowledge_id')) ListenerManagement.embedding_by_problem(args, embedding_model) def embedding_by_data_list(args: List, model_id): - embedding_model = get_embedding_model(model_id) + if not args: + return + from knowledge.serializers.common import get_embedding_model_by_knowledge_id_list + embedding_model = get_embedding_model_by_knowledge_id_list(list({str(row['knowledge_id']) for row in args})) ListenerManagement.embedding_by_data_list(args, embedding_model) @@ -220,7 +241,7 @@ def update_problem_embedding(problem_id: str, problem_content: str, model_id): @param model_id: @return: """ - model = get_embedding_model(model_id) + model = scoped_model(model_id, QuerySet(Problem).filter(id=problem_id).values_list('knowledge_id', flat=True).first()) ListenerManagement.update_problem(UpdateProblemArgs(problem_id, problem_content, model)) @@ -247,8 +268,8 @@ def delete_embedding_by_paragraph_ids(paragraph_ids: List[str]): def update_embedding_document_id(paragraph_id_list, target_document_id, target_knowledge_id, target_embedding_model_id=None): - target_embedding_model = get_embedding_model( - target_embedding_model_id) if target_embedding_model_id is not None else None + target_embedding_model = scoped_model( + target_embedding_model_id, target_knowledge_id) if target_embedding_model_id is not None else None ListenerManagement.update_embedding_document_id( UpdateEmbeddingDocumentIdArgs(paragraph_id_list, target_document_id, target_knowledge_id, target_embedding_model)) diff --git a/apps/knowledge/urls.py b/apps/knowledge/urls.py index 53de5d863..dbde32ae3 100644 --- a/apps/knowledge/urls.py +++ b/apps/knowledge/urls.py @@ -17,6 +17,12 @@ urlpatterns = [ path('workspace//knowledge//sync', views.KnowledgeView.SyncWeb.as_view()), path('workspace//knowledge//generate_related', views.KnowledgeView.GenerateRelated.as_view()), path('workspace//knowledge//embedding', views.KnowledgeView.Embedding.as_view()), + path('workspace//knowledge//metadata', views.KnowledgeView.Metadata.as_view()), + path('workspace//knowledge//parser', views.KnowledgeView.Parser.as_view()), + path('workspace//knowledge//segmentation', views.KnowledgeView.Segmentation.as_view()), + path('workspace//knowledge//rebuild', views.KnowledgeView.Rebuild.as_view()), + path('workspace//knowledge//rebuild/', views.KnowledgeView.RebuildTask.as_view()), + path('workspace//knowledge//rebuild//retry', views.KnowledgeView.RebuildRetry.as_view()), path('workspace//knowledge//hit_test', views.KnowledgeView.HitTest.as_view()), path('workspace//knowledge//export', views.KnowledgeView.Export.as_view()), path('workspace//knowledge//export_zip', views.KnowledgeView.ExportZip.as_view()), diff --git a/apps/knowledge/vector/base_vector.py b/apps/knowledge/vector/base_vector.py index 2a7731fce..6ab7dafaf 100644 --- a/apps/knowledge/vector/base_vector.py +++ b/apps/knowledge/vector/base_vector.py @@ -82,7 +82,7 @@ class BaseVectorStore(ABC): data = {'document_id': document_id, 'paragraph_id': paragraph_id, 'knowledge_id': knowledge_id, 'is_active': is_active, 'source_id': source_id, 'source_type': source_type, 'text': text} chunk_list = chunk_data(data) - result = sub_array(chunk_list) + result = sub_array(chunk_list, self._embedding_batch_size(embedding)) for child_array in result: self._batch_save(child_array, embedding, lambda: False) @@ -96,13 +96,22 @@ class BaseVectorStore(ABC): """ self.save_pre_handler() chunk_list = chunk_data_list(data_list) - result = sub_array(chunk_list) + result = sub_array(chunk_list, self._embedding_batch_size(embedding)) for child_array in result: if not is_the_task_interrupted(): self._batch_save(child_array, embedding, is_the_task_interrupted) else: break + @staticmethod + def _embedding_batch_size(embedding): + """向量批次大小:优先取模型参数 batch_size,默认 10。""" + try: + batch_size = int(getattr(embedding, 'batch_size', 0) or 0) + except (TypeError, ValueError): + batch_size = 0 + return batch_size if batch_size > 0 else 10 + @abstractmethod def _save(self, text, source_type: SourceType, knowledge_id: str, document_id: str, paragraph_id: str, source_id: str, is_active: bool, diff --git a/apps/knowledge/views/document.py b/apps/knowledge/views/document.py index 0db49a6af..b37800899 100644 --- a/apps/knowledge/views/document.py +++ b/apps/knowledge/views/document.py @@ -194,7 +194,7 @@ class DocumentView(APIView): return result.success(DocumentSerializers.Split(data={ 'workspace_id': workspace_id, 'knowledge_id': knowledge_id, - }).parse(split_data)) + }, context={'user': request.user}).parse(split_data)) class SplitPattern(APIView): authentication_classes = [TokenAuth] diff --git a/apps/knowledge/views/knowledge.py b/apps/knowledge/views/knowledge.py index d4420c614..15f467703 100644 --- a/apps/knowledge/views/knowledge.py +++ b/apps/knowledge/views/knowledge.py @@ -1,3 +1,4 @@ +from system_manage.ai_hub_knowledge import parser_binding from django.utils.translation import gettext_lazy as _ from drf_spectacular.utils import extend_schema from rest_framework.request import Request @@ -14,6 +15,8 @@ from knowledge.api.knowledge import KnowledgeBaseCreateAPI, KnowledgeWebCreateAP from knowledge.models import KnowledgeScope from knowledge.serializers.common import get_knowledge_operation_object from knowledge.serializers.knowledge import KnowledgeSerializer +from knowledge.serializers.knowledge_ext import KnowledgeMetadataSerializer, KnowledgeSegmentationSerializer +from knowledge.serializers.rebuild import KnowledgeRebuildSerializer from models_provider.serializers.model_serializer import ModelSerializer @@ -250,6 +253,154 @@ class KnowledgeView(APIView): data={'knowledge_id': knowledge_id, 'workspace_id': workspace_id, 'user_id': request.user.id} ).embedding()) + class Metadata(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="知识库业务元数据", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + def get(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(KnowledgeMetadataSerializer.one(workspace_id, knowledge_id)) + + @extend_schema(methods=['PUT'], summary="修改知识库业务元数据", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + RoleConstants.ADMIN, + ) + @log(menu='Knowledge Base', operate='Modify knowledge metadata', + get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id'))) + def put(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(KnowledgeMetadataSerializer.update( + workspace_id, knowledge_id, request.user, request.data)) + + class Parser(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="知识库解析配置", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + def get(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(parser_binding(workspace_id, knowledge_id, request.user)) + + @extend_schema(methods=['PUT'], summary="修改知识库解析配置", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + RoleConstants.ADMIN, + ) + @log(menu='Knowledge Base', operate='Modify knowledge parser', + get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id'))) + def put(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(parser_binding(workspace_id, knowledge_id, request.user, request.data)) + + class Segmentation(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="知识库分段规则", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + def get(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(KnowledgeSegmentationSerializer.one(workspace_id, knowledge_id)) + + @extend_schema(methods=['PUT'], summary="修改知识库分段规则", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_EDIT.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + @log(menu='Knowledge Base', operate='Modify knowledge segmentation', + get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id'))) + def put(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success(KnowledgeSegmentationSerializer.update( + workspace_id, knowledge_id, request.user, request.data)) + + class Rebuild(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['POST'], summary="独立重建知识库索引", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + @log(menu='Knowledge Base', operate='Rebuild index', + get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id'))) + def post(self, request: Request, workspace_id: str, knowledge_id: str): + return result.success( + KnowledgeRebuildSerializer.create(workspace_id, knowledge_id, str(request.user.id)) + ) + + @extend_schema(methods=['GET'], summary="重建任务列表", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + def get(self, request: Request, workspace_id: str, knowledge_id: str): + current_page = int(request.query_params.get('current_page', 1)) + page_size = int(request.query_params.get('page_size', 10)) + return result.success( + KnowledgeRebuildSerializer.page(workspace_id, knowledge_id, current_page, page_size) + ) + + class RebuildTask(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['GET'], summary="重建任务详情", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + def get(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str): + return result.success( + KnowledgeRebuildSerializer.one(workspace_id, knowledge_id, task_id) + ) + + class RebuildRetry(APIView): + authentication_classes = [TokenAuth] + + @extend_schema(methods=['POST'], summary="重试重建任务", tags=["Knowledge Base"]) + @has_permissions( + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_knowledge_permission(), + PermissionConstants.KNOWLEDGE_VECTOR.get_workspace_permission_workspace_manage_role(), + RoleConstants.WORKSPACE_MANAGE.get_workspace_role(), + ViewPermission([RoleConstants.USER.get_workspace_role()], + [PermissionConstants.KNOWLEDGE.get_workspace_knowledge_permission()], CompareConstants.AND), + ) + @log(menu='Knowledge Base', operate='Retry rebuild index', + get_operation_object=lambda r, k: get_knowledge_operation_object(k.get('knowledge_id'))) + def post(self, request: Request, workspace_id: str, knowledge_id: str, task_id: str): + return result.success( + KnowledgeRebuildSerializer.retry(workspace_id, knowledge_id, task_id, str(request.user.id)) + ) + class Export(APIView): authentication_classes = [TokenAuth] diff --git a/apps/local_model/apps.py b/apps/local_model/apps.py index 285ca7278..270b51c7d 100644 --- a/apps/local_model/apps.py +++ b/apps/local_model/apps.py @@ -4,3 +4,9 @@ from django.apps import AppConfig class LocalModelConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'local_model' + + def ready(self): + import os + if os.environ.get('SERVER_NAME') == 'local_model' and os.environ.get('MAXKB_RUNTIME_AUTOSTART', '1') == '1': + from local_model.runtime import start_runtime + start_runtime() diff --git a/apps/local_model/serializers/model_apply_serializers.py b/apps/local_model/serializers/model_apply_serializers.py index 5ecb2260c..d589c04ad 100644 --- a/apps/local_model/serializers/model_apply_serializers.py +++ b/apps/local_model/serializers/model_apply_serializers.py @@ -7,72 +7,14 @@ @desc: """ import json -import threading -import time -from django.db import connection -from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ from langchain_core.documents import Document from rest_framework import serializers -from local_model.models import Model from local_model.serializers.rsa_util import rsa_long_decrypt from models_provider.impl.local_model_provider.local_model_provider import LocalModelProvider -from common.cache.mem_cache import MemCache - -_lock = threading.Lock() -locks = {} - - -class ModelManage: - cache = MemCache('model', {}) - up_clear_time = time.time() - - @staticmethod - def _get_lock(_id): - lock = locks.get(_id) - if lock is None: - with _lock: - lock = locks.get(_id) - if lock is None: - lock = threading.Lock() - locks[_id] = lock - - return lock - - @staticmethod - def get_model(_id, get_model): - model_instance = ModelManage.cache.get(_id) - if model_instance is None: - lock = ModelManage._get_lock(_id) - with lock: - model_instance = ModelManage.cache.get(_id) - if model_instance is None: - model_instance = get_model(_id) - ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8) - else: - if model_instance.is_cache_model(): - ModelManage.cache.touch(_id, timeout=60 * 60 * 8) - else: - model_instance = get_model(_id) - ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8) - ModelManage.clear_timeout_cache() - return model_instance - - @staticmethod - def clear_timeout_cache(): - if time.time() - ModelManage.up_clear_time > 60 * 60: - threading.Thread(target=lambda: ModelManage.cache.clear_timeout_data()).start() - ModelManage.up_clear_time = time.time() - - @staticmethod - def delete_key(_id): - if ModelManage.cache.has_key(_id): - ModelManage.cache.delete(_id) - - def get_local_model(model, **kwargs): return LocalModelProvider().get_model(model.model_type, model.model_name, json.loads( @@ -81,19 +23,10 @@ def get_local_model(model, **kwargs): streaming=True, **kwargs) -def get_embedding_model(model_id): - model = QuerySet(Model).filter(id=model_id).first() - # 手动关闭数据库连接 - connection.close() - embedding_model = ModelManage.get_model(model_id, - lambda _id: get_local_model(model, use_local=True)) - return embedding_model - - class EmbedDocuments(serializers.Serializer): texts = serializers.ListField(required=True, child=serializers.CharField(required=True, label=_('vector text')), - label=_('vector text list')), + label=_('vector text list')) class EmbedQuery(serializers.Serializer): @@ -133,28 +66,33 @@ class ModelApplySerializers(serializers.Serializer): self.is_valid(raise_exception=True) EmbedDocuments(data=instance).is_valid(raise_exception=True) - model = get_embedding_model(self.data.get('model_id')) - return model.embed_documents(instance.getlist('texts')) + from local_model.runtime import manager + with manager.inference(self.data.get('model_id')) as model: + texts = instance.getlist('texts') if hasattr(instance, 'getlist') else instance.get('texts') + return model.embed_documents(texts) def embed_query(self, instance, with_valid=True): if with_valid: self.is_valid(raise_exception=True) EmbedQuery(data=instance).is_valid(raise_exception=True) - model = get_embedding_model(self.data.get('model_id')) - return model.embed_query(instance.get('text')) + from local_model.runtime import manager + with manager.inference(self.data.get('model_id')) as model: + return model.embed_query(instance.get('text')) def compress_documents(self, instance, with_valid=True): if with_valid: self.is_valid(raise_exception=True) CompressDocuments(data=instance).is_valid(raise_exception=True) - model = get_embedding_model(self.data.get('model_id')) - return [{'page_content': d.page_content, 'metadata': d.metadata} for d in model.compress_documents( - [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document in - instance.get('documents')], instance.get('query'))] + from local_model.runtime import manager + with manager.inference(self.data.get('model_id')) as model: + return [{'page_content': d.page_content, 'metadata': d.metadata} for d in model.compress_documents( + [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document in + instance.get('documents')], instance.get('query'))] def unload(self, with_valid=True): if with_valid: self.is_valid(raise_exception=True) - ModelManage.delete_key(self.data.get('model_id')) + from local_model.runtime import manager + manager.unload(self.data.get('model_id')) return True diff --git a/apps/local_model/urls.py b/apps/local_model/urls.py index a9c060254..36720d182 100644 --- a/apps/local_model/urls.py +++ b/apps/local_model/urls.py @@ -3,10 +3,12 @@ import os from django.urls import path from . import views +from .views.runtime import LocalModelRuntime app_name = "local_model" # @formatter:off urlpatterns = [ + path('model_runtime', LocalModelRuntime.as_view()), path('model/validate', views.LocalModelApply.Validate.as_view()), path('model//embed_documents', views.LocalModelApply.EmbedDocuments.as_view()), path('model//embed_query', views.LocalModelApply.EmbedQuery.as_view()), diff --git a/apps/local_model/views/model_apply.py b/apps/local_model/views/model_apply.py index 98c07dd74..b2cfe72d9 100644 --- a/apps/local_model/views/model_apply.py +++ b/apps/local_model/views/model_apply.py @@ -35,8 +35,9 @@ class LocalModelApply(APIView): class Unload(APIView): def post(self, request: Request, model_id): - return result.success( - ModelApplySerializers(data={'model_id': model_id}).compress_documents(request.data)) + # Administrative weight operations use the signed runtime endpoint. + from rest_framework.exceptions import PermissionDenied + raise PermissionDenied('请通过模型管理卸载') class Validate(APIView): def post(self, request: Request): diff --git a/apps/maxkb/conf.py b/apps/maxkb/conf.py index da060f955..c73977eda 100644 --- a/apps/maxkb/conf.py +++ b/apps/maxkb/conf.py @@ -46,7 +46,10 @@ class Config(dict): } def get_debug(self) -> bool: - return self.get('DEBUG') if 'DEBUG' in self else True + value = self.get('DEBUG', True) + if isinstance(value, str): + return value.strip().lower() in ('1', 'true', 'yes', 'on') + return bool(value) def get_time_zone(self) -> str: return self.get('TIME_ZONE') if 'TIME_ZONE' in self else 'Asia/Shanghai' diff --git a/apps/maxkb/settings/base/web.py b/apps/maxkb/settings/base/web.py index 04f6ae5a3..0f6991e7f 100644 --- a/apps/maxkb/settings/base/web.py +++ b/apps/maxkb/settings/base/web.py @@ -62,6 +62,7 @@ MIDDLEWARE = [ REST_FRAMEWORK = { 'EXCEPTION_HANDLER': 'common.exception.handle_exception.handle_exception', 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', + 'UNAUTHENTICATED_USER': None, 'DEFAULT_AUTHENTICATION_CLASSES': ['common.auth.authenticate.AnonymousAuthentication'] } STATICFILES_DIRS = [(os.path.join(PROJECT_DIR, 'ui', 'dist'))] diff --git a/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/llm.py b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/llm.py index 53d9ebda9..9c99153a3 100644 --- a/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/llm.py +++ b/apps/models_provider/impl/aliyun_bai_lian_model_provider/credential/llm.py @@ -25,6 +25,20 @@ class BaiLianLLMModelParams(BaseForm): precision=2 ) + top_p = forms.SliderField( + TooltipLabel( + _('TopP'), + _('Nucleus sampling: the model considers the results of the top_p probability mass of tokens. ' + 'The smaller the value, the more deterministic the generation') + ), + required=True, + default_value=0.8, + _min=0, + _max=1, + _step=0.01, + precision=2 + ) + max_tokens = forms.SliderField( TooltipLabel( _('Output the maximum Tokens'), diff --git a/apps/models_provider/impl/local_model_provider/credential/embedding/model.py b/apps/models_provider/impl/local_model_provider/credential/embedding/model.py index d9ec4c3da..e174062d9 100644 --- a/apps/models_provider/impl/local_model_provider/credential/embedding/model.py +++ b/apps/models_provider/impl/local_model_provider/credential/embedding/model.py @@ -13,7 +13,7 @@ from django.utils.translation import gettext_lazy as _, gettext from common import forms from common.exception.app_exception import AppApiException -from common.forms import BaseForm +from common.forms import BaseForm, TooltipLabel from models_provider.base_model_provider import BaseModelCredential, ValidCode from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding @@ -51,3 +51,29 @@ class LocalEmbeddingCredential(BaseForm, BaseModelCredential): return model cache_folder = forms.TextInputField(_('Model catalog'), required=True) + device = forms.SingleSelect( + TooltipLabel(_('Device'), _('Runtime device for the embedding model')), + text_field='label', + value_field='value', + option_list=[ + {'value': 'cpu', 'label': 'CPU'}, + {'value': 'cuda', 'label': 'GPU(CUDA)'}, + ], + required=False, + default_value='cpu' + ) + normalize = forms.SwitchField( + TooltipLabel(_('Normalize'), _('Whether to normalize embeddings to unit length')), + required=False, + default_value=True, + attrs={"active-value": "true", "inactive-value": "false"} + ) + batch_size = forms.SliderField( + TooltipLabel(_('Batch size'), _('Number of texts per embedding request')), + required=False, + default_value=10, + _min=1, + _max=64, + _step=1, + precision=0 + ) diff --git a/apps/models_provider/impl/local_model_provider/model/embedding/model.py b/apps/models_provider/impl/local_model_provider/model/embedding/model.py index a9e7aad69..ac4f9b34b 100644 --- a/apps/models_provider/impl/local_model_provider/model/embedding/model.py +++ b/apps/models_provider/impl/local_model_provider/model/embedding/model.py @@ -15,6 +15,7 @@ from models_provider.base_model_provider import MaxKBBaseModel class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings): + batch_size: int = 10 @staticmethod def is_cache_model(): return True @@ -27,7 +28,15 @@ class LocalEmbedding(MaxKBBaseModel, HuggingFaceEmbeddings): local_model_path = os.path.join(cache_folder, model_name) if os.path.isdir(local_model_path): model_path = local_model_path - return LocalEmbedding(model_name=model_path, cache_folder=cache_folder, - model_kwargs={'device': model_credential.get('device')}, - encode_kwargs={'normalize_embeddings': True} - ) + normalize = model_credential.get('normalize', True) + if isinstance(normalize, str): + normalize = normalize.lower() == 'true' + instance = LocalEmbedding(model_name=model_path, cache_folder=cache_folder, + model_kwargs={'device': model_credential.get('device')}, + encode_kwargs={'normalize_embeddings': normalize if normalize is not None else True} + ) + try: + instance.batch_size = int(model_credential.get('batch_size') or 10) + except (TypeError, ValueError): + instance.batch_size = 10 + return instance diff --git a/apps/models_provider/impl/local_model_provider/model/embedding/web.py b/apps/models_provider/impl/local_model_provider/model/embedding/web.py index 42c780adc..6fe594900 100644 --- a/apps/models_provider/impl/local_model_provider/model/embedding/web.py +++ b/apps/models_provider/impl/local_model_provider/model/embedding/web.py @@ -24,19 +24,25 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings): @staticmethod def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): + try: + batch_size = int(model_credential.get('batch_size') or 10) + except (TypeError, ValueError): + batch_size = 10 return LocalEmbedding(model_name=model_name, cache_folder=model_credential.get('cache_folder'), model_kwargs={'device': model_credential.get('device')}, encode_kwargs={'normalize_embeddings': True}, + batch_size=batch_size, **model_kwargs) model_id: str = None + batch_size: int = 10 def embed_query(self, text: str) -> List[float]: bind = f'{CONFIG.get("LOCAL_MODEL_HOST")}:{CONFIG.get("LOCAL_MODEL_PORT")}' prefix = CONFIG.get_admin_path() res = requests.post( f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_query', - {'text': text}) + {'text': text}, timeout=(5, 620)) result = res.json() if result.get('code', 500) == 200: return result.get('data') @@ -47,7 +53,7 @@ class LocalEmbedding(MaxKBBaseModel, BaseModel, Embeddings): prefix = CONFIG.get_admin_path() res = requests.post( f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/embed_documents', - {'texts': texts}) + {'texts': texts}, timeout=(5, 620)) result = res.json() if result.get('code', 500) == 200: return result.get('data') diff --git a/apps/models_provider/impl/local_model_provider/model/reranker/model.py b/apps/models_provider/impl/local_model_provider/model/reranker/model.py index a66776101..ac660dd6d 100644 --- a/apps/models_provider/impl/local_model_provider/model/reranker/model.py +++ b/apps/models_provider/impl/local_model_provider/model/reranker/model.py @@ -8,6 +8,7 @@ """ from typing import Sequence, Optional, Dict, Any +import os from langchain_core.callbacks import Callbacks from langchain_core.documents import Document, BaseDocumentCompressor @@ -39,7 +40,10 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor): @staticmethod def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): - return LocalReranker(model_name, cache_dir=model_credential.get('cache_dir')) + cache_dir = model_credential.get('cache_dir') or model_credential.get('cache_folder') + if cache_dir and not os.path.isabs(model_name) and os.path.isdir(os.path.join(cache_dir, model_name)): + model_name = os.path.join(cache_dir, model_name) + return LocalReranker(model_name, cache_dir=cache_dir, device=model_credential.get('device') or 'cpu') def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \ Sequence[Document]: @@ -48,7 +52,7 @@ class LocalReranker(MaxKBBaseModel, BaseDocumentCompressor): import torch with torch.no_grad(): inputs = self.tokenizer([[query, document.page_content] for document in documents], padding=True, - truncation=True, return_tensors='pt', max_length=512) + truncation=True, return_tensors='pt', max_length=512).to(self.client.device) scores = [torch.sigmoid(s).float().item() for s in self.client(**inputs, return_dict=True).logits.view(-1, ).float()] result = [Document(page_content=documents[index].page_content, metadata={'relevance_score': scores[index]}) diff --git a/apps/models_provider/impl/local_model_provider/model/reranker/web.py b/apps/models_provider/impl/local_model_provider/model/reranker/web.py index 45ab6978a..eeb051c98 100644 --- a/apps/models_provider/impl/local_model_provider/model/reranker/web.py +++ b/apps/models_provider/impl/local_model_provider/model/reranker/web.py @@ -32,7 +32,6 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor): def __init__(self, **kwargs): super().__init__(**kwargs) - print('ssss', kwargs.get('model_id', None)) self.model_id = kwargs.get('model_id', None) def compress_documents(self, documents: Sequence[Document], query: str, callbacks: Optional[Callbacks] = None) -> \ @@ -44,7 +43,7 @@ class LocalReranker(MaxKBBaseModel, BaseModel, BaseDocumentCompressor): res = requests.post( f'{CONFIG.get("LOCAL_MODEL_PROTOCOL")}://{bind}{prefix}/api/model/{self.model_id}/compress_documents', json={'documents': [{'page_content': document.page_content, 'metadata': document.metadata} for document in - documents], 'query': query}, headers={'Content-Type': 'application/json'}) + documents], 'query': query}, headers={'Content-Type': 'application/json'}, timeout=(5, 620)) result = res.json() if result.get('code', 500) == 200: return [Document(page_content=document.get('page_content'), metadata=document.get('metadata')) for document diff --git a/apps/models_provider/impl/ollama_model_provider/credential/embedding.py b/apps/models_provider/impl/ollama_model_provider/credential/embedding.py index 15a869a45..9e23c18f0 100644 --- a/apps/models_provider/impl/ollama_model_provider/credential/embedding.py +++ b/apps/models_provider/impl/ollama_model_provider/credential/embedding.py @@ -12,7 +12,7 @@ from django.utils.translation import gettext as _ from common import forms from common.exception.app_exception import AppApiException -from common.forms import BaseForm +from common.forms import BaseForm, TooltipLabel from models_provider.base_model_provider import BaseModelCredential, ValidCode from models_provider.impl.local_model_provider.model.embedding import LocalEmbedding @@ -47,3 +47,12 @@ class OllamaEmbeddingModelCredential(BaseForm, BaseModelCredential): return self api_base = forms.TextInputField('API URL', required=True) + batch_size = forms.SliderField( + TooltipLabel(_('Batch size'), _('Number of texts per embedding request')), + required=False, + default_value=10, + _min=1, + _max=64, + _step=1, + precision=0 + ) diff --git a/apps/models_provider/impl/ollama_model_provider/model/embedding.py b/apps/models_provider/impl/ollama_model_provider/model/embedding.py index 8454aafe9..809651ee2 100644 --- a/apps/models_provider/impl/ollama_model_provider/model/embedding.py +++ b/apps/models_provider/impl/ollama_model_provider/model/embedding.py @@ -14,12 +14,26 @@ from models_provider.base_model_provider import MaxKBBaseModel class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings): + model_resource_id: str | None = None + runtime_keep_alive: int = 300 + batch_size: int = 10 + + @property + def _default_params(self): + return {**super()._default_params, 'keep_alive': self.runtime_keep_alive} @staticmethod def new_instance(model_type, model_name, model_credential: Dict[str, object], **model_kwargs): - return OllamaEmbedding( + try: + batch_size = int(model_credential.get('batch_size') or 10) + except (TypeError, ValueError): + batch_size = 10 + instance = OllamaEmbedding( model=model_name, base_url=model_credential.get('api_base'), + model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None, ) + instance.batch_size = batch_size + return instance def embed_documents(self, texts: List[str]) -> List[List[float]]: """Embed documents using an Ollama deployed embedding model. @@ -31,8 +45,13 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings): List of embeddings, one for each text. """ instruction_pairs = [f"{text}" for text in texts] - embeddings = self._embed(instruction_pairs) - return embeddings + if not texts: + return [] + from common.model_runtime_client import inference_lease + with inference_lease(self.model_resource_id) as state: + if state: + self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds'] + return self._embed(instruction_pairs) def embed_query(self, text: str) -> List[float]: """Embed a query using a Ollama deployed embedding model. @@ -44,5 +63,8 @@ class OllamaEmbedding(MaxKBBaseModel, OllamaEmbeddings): Embeddings for the text. """ instruction_pair = f"{text}" - embedding = self._embed([instruction_pair])[0] - return embedding + from common.model_runtime_client import inference_lease + with inference_lease(self.model_resource_id) as state: + if state: + self.runtime_keep_alive = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds'] + return self._embed([instruction_pair])[0] diff --git a/apps/models_provider/impl/ollama_model_provider/model/llm.py b/apps/models_provider/impl/ollama_model_provider/model/llm.py index 047582426..6a2db1aa9 100644 --- a/apps/models_provider/impl/ollama_model_provider/model/llm.py +++ b/apps/models_provider/impl/ollama_model_provider/model/llm.py @@ -26,6 +26,7 @@ def get_base_url(url: str): class OllamaChatModel(MaxKBBaseModel, ChatOllama): + model_resource_id: str | None = None @staticmethod def is_cache_model(): return False @@ -37,8 +38,24 @@ class OllamaChatModel(MaxKBBaseModel, ChatOllama): optional_params = MaxKBBaseModel.filter_optional_params(model_kwargs) return OllamaChatModel(model=model_name, base_url=base_url, + model_resource_id=str(model_kwargs['model_id']) if model_kwargs.get('model_id') else None, stream=True, **optional_params) + def _create_chat_stream(self, messages, stop=None, **kwargs): + from common.model_runtime_client import inference_lease + with inference_lease(self.model_resource_id) as state: + if state: + kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds'] + yield from super()._create_chat_stream(messages, stop, **kwargs) + + async def _acreate_chat_stream(self, messages, stop=None, **kwargs): + from common.model_runtime_client import async_inference_lease + async with async_inference_lease(self.model_resource_id) as state: + if state: + kwargs['keep_alive'] = -1 if state['strategy'] == 'PRELOAD' else state['idle_seconds'] + async for part in super()._acreate_chat_stream(messages, stop, **kwargs): + yield part + def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: tokenizer = TokenizerManage.get_tokenizer() return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages]) diff --git a/apps/models_provider/urls.py b/apps/models_provider/urls.py index 4728c44f2..ee79e07b7 100644 --- a/apps/models_provider/urls.py +++ b/apps/models_provider/urls.py @@ -3,10 +3,14 @@ import os from django.urls import path from . import views +from .views.local_imports import LocalModelImports +from .views.runtime import ModelLoading app_name = "models_provider" # @formatter:off urlpatterns = [ + path('workspace//model//runtime', ModelLoading.as_view()), + path('local_model/imports', LocalModelImports.as_view()), path('provider', views.Provide.as_view()), path('provider/model_type_list', views.Provide.ModelTypeList.as_view()), path('provider/model_list', views.Provide.ModelList.as_view()), diff --git a/apps/ops/celery/__init__.py b/apps/ops/celery/__init__.py index 53dac8cc1..25a6d6c90 100644 --- a/apps/ops/celery/__init__.py +++ b/apps/ops/celery/__init__.py @@ -21,7 +21,8 @@ configs['worker_concurrency'] = 5 configs["task_queues"] = [ Queue("celery", Exchange("celery"), routing_key="celery"), - Queue("model", Exchange("model"), routing_key="model") + Queue("model", Exchange("model"), routing_key="model"), + Queue("uniskb_knowledge", Exchange("uniskb_knowledge"), routing_key="uniskb_knowledge") ] app.namespace = 'CELERY' app.conf.update( @@ -30,4 +31,14 @@ app.conf.update( key) for key in configs.keys()}) +# 新知识库任务与共享 Redis 中的旧 celery 队列隔离。 +knowledge_routes = { + "celery:embedding_*": {"queue": "uniskb_knowledge"}, + "celery:rebuild_knowledge_index": {"queue": "uniskb_knowledge"}, + "celery:finalize_rebuild_task": {"queue": "uniskb_knowledge"}, +} +existing_routes = app.conf.task_routes or () +if isinstance(existing_routes, dict): + existing_routes = (existing_routes,) +app.conf.task_routes = (knowledge_routes, *existing_routes) app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS]) diff --git a/apps/system_manage/models/__init__.py b/apps/system_manage/models/__init__.py index 2141e3d68..b5f3fd7d8 100644 --- a/apps/system_manage/models/__init__.py +++ b/apps/system_manage/models/__init__.py @@ -12,3 +12,4 @@ from .log_management import * from .chat_user import * from .user_menu_setting import * from .access_control_policy import * +from .ai_hub import * diff --git a/apps/system_manage/serializers/system.py b/apps/system_manage/serializers/system.py index f3e1e096d..fe507e34e 100644 --- a/apps/system_manage/serializers/system.py +++ b/apps/system_manage/serializers/system.py @@ -48,6 +48,9 @@ class SystemProfileSerializer(serializers.Serializer): class SystemConfigSerializer(serializers.Serializer): @staticmethod def get_config(): + # Prefer the bundled workbench: it uses the current login through its parent. + bundled = os.path.join(settings.STATIC_ROOT, 'admin', 'mindmap-workbench', 'index.html') return { - 'mindmap_url': CONFIG.get('MINDMAP_URL', '') + 'mindmap_url': (CONFIG.get_admin_path() + '/mindmap-workbench/index.html') + if os.path.isfile(bundled) else CONFIG.get('MINDMAP_URL', '') } diff --git a/apps/system_manage/serializers/user_resource_permission.py b/apps/system_manage/serializers/user_resource_permission.py index b34b8544e..9ce375ec3 100644 --- a/apps/system_manage/serializers/user_resource_permission.py +++ b/apps/system_manage/serializers/user_resource_permission.py @@ -31,7 +31,8 @@ from maxkb.settings import edition from models_provider.models import Model from system_manage.models import WorkspaceUserResourcePermission from tools.models import Tool -from users.serializers.user import is_workspace_manage +from users.models import User +from users.serializers.user import is_workspace_manage, exclude_unactivated_users class PermissionSerializer(serializers.Serializer): @@ -108,6 +109,13 @@ class UserResourcePermissionSerializer(serializers.Serializer): user_id = serializers.CharField(required=True, label=_('user id')) auth_target_type = serializers.CharField(required=True, label=_('resource')) + def is_valid(self, *, raise_exception=False): + valid = super().is_valid(raise_exception=raise_exception) + if valid and not exclude_unactivated_users(User.objects.filter( + id=self.validated_data['user_id'])).exists(): + raise AppApiException(500, '仅可为已激活且启用的用户授权') + return valid + def get_queryset(self, instance): resource_query_set = QuerySet( model=get_dynamics_model({ @@ -345,6 +353,9 @@ class ResourceUserPermissionSerializer(serializers.Serializer): "user_role_relation.workspace_id": models.CharField(), })) + user_query_set = user_query_set.filter(**{ + 'u.id__in': exclude_unactivated_users(User.objects.all()).values('id') + }) nick_name = instance.get('nick_name') username = instance.get('username') permission = instance.get('permission') @@ -458,6 +469,9 @@ class ResourceUserPermissionSerializer(serializers.Serializer): users_permission = instance users_id = [item["user_id"] for item in users_permission] + eligible_ids = exclude_unactivated_users(User.objects.filter(id__in=users_id)) + if eligible_ids.count() != len(set(users_id)): + raise AppApiException(500, '仅可为已激活且启用的用户授权') include_children = users_permission[0].get('include_children') folder_ids = users_permission[0].get('folder_ids') # 删除已存在的对应的用户在该资源下的权限 diff --git a/apps/system_manage/urls.py b/apps/system_manage/urls.py index 3c0051f32..4d60a11cc 100644 --- a/apps/system_manage/urls.py +++ b/apps/system_manage/urls.py @@ -1,10 +1,15 @@ from django.urls import path from . import views +from .views.ai_hub import HubView, PublishedRun, ParserTest app_name = "system_manage" # @formatter:off urlpatterns = [ + path('workspace//ai_hub/parsers//test', ParserTest.as_view()), + path('workspace//ai_hub/published//run', PublishedRun.as_view()), + path('workspace//ai_hub/', HubView.as_view()), + path('workspace//ai_hub//', HubView.as_view()), path('workspace//user_resource_permission/user//resource/', views.WorkSpaceUserResourcePermissionView.as_view()), path('workspace//user_resource_permission/user//resource///', views.WorkSpaceUserResourcePermissionView.Page.as_view()), path('workspace//resource_user_permission/resource//resource/', views.WorkspaceResourceUserPermissionView.as_view()), diff --git a/apps/users/models/__init__.py b/apps/users/models/__init__.py index 5708f6a24..f0a8102d3 100644 --- a/apps/users/models/__init__.py +++ b/apps/users/models/__init__.py @@ -7,3 +7,4 @@ @desc: """ from .user import * +from .user_manage_ext import * diff --git a/apps/users/serializers/login.py b/apps/users/serializers/login.py index e5487f137..2897de086 100644 --- a/apps/users/serializers/login.py +++ b/apps/users/serializers/login.py @@ -25,7 +25,7 @@ from common.exception.app_exception import AppApiException from common.utils.common import password_encrypt, get_random_chars from common.utils.rsa_util import encrypt, decrypt from maxkb.const import CONFIG -from users.models import User +from users.models import User, UserActivationApplication, ActivationStatus class LoginRequest(serializers.Serializer): @@ -117,6 +117,12 @@ class LoginSerializer(serializers.Serializer): record_login_fail(username) raise AppApiException(500, _('The username or password is incorrect')) if not user.is_active: + application = UserActivationApplication.objects.filter(user_id=user.id).first() + if application and application.status == ActivationStatus.PENDING: + raise AppApiException(1005, '账号待审核') + if application and application.status == ActivationStatus.REJECTED: + reject_reason = application.reject_reason or '不符合激活条件' + raise AppApiException(1005, f'注册申请未通过,拒绝原因:{reject_reason}') record_login_fail(username) raise AppApiException(1005, _("The user has been disabled, please contact the administrator!")) cache.delete(system_get_key(f'system_{username}'), version=system_version) diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index 04be65117..e01ad1f68 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -28,7 +28,7 @@ from common.exception.app_exception import AppApiException from common.utils.common import valid_license, password_encrypt from maxkb.conf import PROJECT_DIR from system_manage.models import SystemSetting, SettingType -from users.models import User +from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment from django.utils.translation import gettext_lazy as _, to_locale from django.core import validators from django.core.mail import send_mail @@ -46,6 +46,18 @@ PASSWORD_REGEX = re.compile( version, get_key = Cache_Version.SYSTEM.value +def exclude_unactivated_users(query_set): + """仅保留“激活审核通过”的用户。 + + 排除存在待激活申请且状态为 待审核/已拒绝 的用户,同时要求账号处于启用状态; + 未走激活审核流程的管理员创建用户不受影响。用于资源授权、角色/成员选择等场景。 + """ + unapproved_user_ids = UserActivationApplication.objects.filter( + status__in=[ActivationStatus.PENDING, ActivationStatus.REJECTED] + ).values('user_id') + return query_set.exclude(id__in=unapproved_user_ids).filter(is_active=True) + + class UserProfileResponse(serializers.ModelSerializer): is_edit_password = serializers.BooleanField(required=True, label=_('Is Edit Password')) permissions = serializers.ListField(required=True, label=_('permissions')) @@ -180,6 +192,11 @@ class UserManageSerializer(serializers.Serializer): max_length=20, default="LOCAL" ) + pending_activation = serializers.BooleanField( + required=False, + label="创建后待激活", + default=True + ) def is_valid(self, *, raise_exception=True): @@ -233,7 +250,10 @@ class UserManageSerializer(serializers.Serializer): email = self.data.get('email') is_active = self.data.get('is_active', None) source = self.data.get('source', None) - query_set = QuerySet(User) + unapproved_user_ids = UserActivationApplication.objects.filter( + status__in=[ActivationStatus.PENDING, ActivationStatus.REJECTED] + ).values('user_id') + query_set = QuerySet(User).exclude(id__in=unapproved_user_ids) if username is not None: query_set = query_set.filter(username__contains=username) if nick_name is not None: @@ -331,8 +351,11 @@ class UserManageSerializer(serializers.Serializer): @transaction.atomic def save(self, instance, user_id, with_valid=True): if with_valid: - self.UserInstance(data=instance).is_valid(raise_exception=True) + serializer = self.UserInstance(data=instance) + serializer.is_valid(raise_exception=True) + instance = {**instance, **serializer.validated_data} + pending_activation = bool(instance.get('pending_activation', True)) user = User( id=uuid.uuid7(), email=instance.get('email'), @@ -342,10 +365,20 @@ class UserManageSerializer(serializers.Serializer): password=password_encrypt(instance.get('password')), role=RoleConstants.USER.name, source=instance.get('source', 'LOCAL'), - is_active=True + is_active=not pending_activation ) update_user_role(instance, user, user_id) user.save() + if pending_activation: + UserActivationApplication.objects.create( + user=user, + email=user.email or '', + email_verified=False, + status=ActivationStatus.PENDING + ) + else: + from users.serializers.user_manage_ext import send_activation_notice + transaction.on_commit(lambda: send_activation_notice(user, True)) return UserInstanceSerializer(user).data class UserEditInstance(serializers.Serializer): @@ -452,6 +485,8 @@ class UserManageSerializer(serializers.Serializer): raise AppApiException(1004, _('Unable to delete administrator')) def edit(self, instance, user_id, with_valid=True): + if 'role' in instance or 'role_setting' in instance: + raise AppApiException(500, '请通过分配角色修改角色') if with_valid: self.is_valid(raise_exception=True) UserManageSerializer.UserEditInstance(data=instance).is_valid(user_id=self.data.get('id'), @@ -459,7 +494,6 @@ class UserManageSerializer(serializers.Serializer): user = User.objects.filter(id=self.data.get('id')).first() self._check_admin_modification(user, instance) self._update_user_fields(user, instance) - update_user_role(instance, user, user_id) user.save() return UserInstanceSerializer(user).data @@ -474,7 +508,7 @@ class UserManageSerializer(serializers.Serializer): @staticmethod def _update_user_fields(user, instance): - update_keys = ['email', 'nick_name', 'phone', 'is_active', 'role'] + update_keys = ['email', 'nick_name', 'phone', 'is_active'] for key in update_keys: if key in instance and instance.get(key) is not None: setattr(user, key, instance.get(key)) @@ -531,7 +565,7 @@ class UserManageSerializer(serializers.Serializer): else: user_ids = User.objects.values_list('id', flat=True) - users = User.objects.filter(id__in=user_ids).values('id', 'nick_name') + users = exclude_unactivated_users(User.objects.filter(id__in=user_ids)).values('id', 'nick_name') return list(users) def get_user_members(self, workspace_id): @@ -549,6 +583,8 @@ class UserManageSerializer(serializers.Serializer): .filter(workspace_id=workspace_id, role__type='USER') .select_related('role', 'user') ) + eligible_user_ids = exclude_unactivated_users(User.objects.all()).values_list('id', flat=True) + user_role_relations = user_role_relations.filter(user_id__in=eligible_user_ids) user_dict = {} for relation in user_role_relations: user_id = relation.user.id @@ -564,7 +600,7 @@ class UserManageSerializer(serializers.Serializer): # 将字典值转换为列表形式 return list(user_dict.values()) - user_list = User.objects.exclude(role=RoleConstants.ADMIN.name) + user_list = exclude_unactivated_users(User.objects.exclude(role=RoleConstants.ADMIN.name)) return [ { 'id': user.id, @@ -587,7 +623,7 @@ class UserManageSerializer(serializers.Serializer): return True def get_all_user_list(self): - users = User.objects.all().values('id', 'nick_name', 'username') + users = exclude_unactivated_users(User.objects.all()).values('id', 'nick_name', 'username') return list(users) diff --git a/apps/users/tests.py b/apps/users/tests.py index 7ce503c2d..38dcdc4f0 100644 --- a/apps/users/tests.py +++ b/apps/users/tests.py @@ -1,3 +1,418 @@ from django.test import TestCase +from django.conf import settings +from django.core.cache import cache +from unittest.mock import patch +import json -# Create your tests here. +from rest_framework.exceptions import ValidationError +from rest_framework.test import APIRequestFactory + +from common.constants.permission_constants import RoleConstants +from common.constants.cache_version import Cache_Version +from common.exception.app_exception import AppApiException +from common.utils.common import password_encrypt +from common.utils.rsa_util import encrypt, generate +from users.models import User, UserActivationApplication, ActivationStatus, UserRoleAssignment +from users.serializers.login import LoginSerializer +from users.serializers.register import RegisterSerializer +from users.serializers.user import UserManageSerializer +from users.serializers.user_manage_ext import ActivationSerializer, RoleAssignmentSerializer, send_activation_notice +from users.serializers.user import exclude_unactivated_users +from system_manage.serializers.user_resource_permission import UserResourcePermissionSerializer, ResourceUserPermissionSerializer +from unittest.mock import Mock +from users.views.register import RegisterView +from users.views.user_manage_ext import UserActivationApproveView + + +@patch('users.serializers.user.DatabaseModelManage.get_model', return_value=None) +class RegistrationTests(TestCase): + def setUp(self): + self.payload = { + 'username': 'signup_test', 'nick_name': '注册测试', + 'email': 'signup@example.com', 'phone': '', + 'password': 'Signup@123', 're_password': 'Signup@123', + } + self.admin = User.objects.create( + username='review_admin', nick_name='审核员', email='review@example.com', + password=password_encrypt('Review@123'), role=RoleConstants.ADMIN.name, + ) + + def test_public_registration_forces_pending_local_user(self, _registry): + response = RegisterView.as_view()(APIRequestFactory().post('/user/register', { + **self.payload, 'is_active': True, 'pending_activation': False, + 'role': 'ADMIN', 'source': 'LDAP', + 'role_setting': [{'role_id': 'ADMIN', 'workspace_ids': ['default']}], + }, format='json')) + self.assertEqual(response.status_code, 200) + user = User.objects.get(username=self.payload['username']) + self.assertFalse(user.is_active) + self.assertEqual(user.role, 'USER') + self.assertEqual(user.source, 'LOCAL') + self.assertEqual(user.password, password_encrypt(self.payload['password'])) + self.assertEqual(user.activation_application.status, ActivationStatus.PENDING) + self.assertFalse(UserRoleAssignment.objects.filter(user=user).exists()) + + def test_password_confirmation_is_required(self, _registry): + with self.assertRaises(ValidationError): + RegisterSerializer.register({**self.payload, 're_password': 'Other@123'}) + self.assertFalse(User.objects.filter(username=self.payload['username']).exists()) + + def test_encrypted_registration_credentials(self, _registry): + key_pair = generate() + credentials = {key: self.payload[key] for key in ('password', 're_password')} + payload = {key: value for key, value in self.payload.items() if key not in credentials} + payload['encryptedData'] = encrypt(json.dumps(credentials), public_key=key_pair['key']) + with patch('common.utils.rsa_util.get_key_pair', return_value=key_pair): + RegisterSerializer.register(payload) + user = User.objects.get(username=self.payload['username']) + self.assertEqual(user.password, password_encrypt(self.payload['password'])) + self.assertFalse(user.is_active) + + def test_anonymous_user_cannot_approve(self, _registry): + RegisterSerializer.register(self.payload) + application = UserActivationApplication.objects.get() + response = UserActivationApproveView.as_view()( + APIRequestFactory().post('/activation/approve', {}, format='json'), + application_id=str(application.id), + ) + self.assertEqual(response.status_code, 401) + application.refresh_from_db() + self.assertEqual(application.status, ActivationStatus.PENDING) + + def test_invalid_email_and_weak_password_are_rejected(self, _registry): + for change in ({'email': 'invalid'}, {'password': '123456', 're_password': '123456'}): + with self.subTest(change=list(change)): + with self.assertRaises(ValidationError): + RegisterSerializer.register({**self.payload, **change}) + + def test_duplicate_registration_does_not_create_another_application(self, _registry): + RegisterSerializer.register(self.payload) + with self.assertRaises(AppApiException): + RegisterSerializer.register(self.payload) + self.assertEqual(UserActivationApplication.objects.count(), 1) + + @patch.object(LoginSerializer, 'get_auth_setting', return_value={'max_attempts': 1}) + def test_login_requires_approval(self, _settings, _registry): + RegisterSerializer.register(self.payload) + credentials = {'username': self.payload['username'], 'password': self.payload['password']} + with self.assertRaises(AppApiException) as error: + LoginSerializer.login(credentials.copy()) + self.assertIn('待审核', str(error.exception)) + application = UserActivationApplication.objects.get() + ActivationSerializer.approve(str(application.id), str(self.admin.id)) + self.assertIn('token', LoginSerializer.login(credentials.copy())) + application.refresh_from_db() + self.assertEqual(application.status, ActivationStatus.APPROVED) + self.assertEqual(application.audit_by_id, self.admin.id) + self.assertTrue(UserRoleAssignment.objects.filter(user=application.user, role_id='USER').exists()) + with self.assertRaises(AppApiException): + ActivationSerializer.approve(str(application.id), str(self.admin.id)) + + @patch.object(LoginSerializer, 'get_auth_setting', return_value={'max_attempts': -1}) + def test_rejected_registration_cannot_login(self, _settings, _registry): + RegisterSerializer.register(self.payload) + application = UserActivationApplication.objects.get() + ActivationSerializer.reject(str(application.id), str(self.admin.id), '审核未通过') + with self.assertRaises(AppApiException) as error: + LoginSerializer.login({'username': self.payload['username'], 'password': self.payload['password']}) + self.assertIn('未通过', str(error.exception)) + application.user.refresh_from_db() + self.assertFalse(application.user.is_active) + + def test_admin_creation_can_activate_immediately(self, _registry): + data = UserManageSerializer().save({**self.payload, 'pending_activation': False}, str(self.admin.id)) + self.assertTrue(data['is_active']) + self.assertFalse(UserActivationApplication.objects.filter(user_id=data['id']).exists()) + + def test_admin_creation_can_require_approval(self, _registry): + data = UserManageSerializer().save({**self.payload, 'pending_activation': True}, str(self.admin.id)) + self.assertFalse(data['is_active']) + self.assertTrue(UserActivationApplication.objects.filter(user_id=data['id'], status='PENDING').exists()) + + def test_admin_creation_defaults_to_pending(self, _registry): + data = UserManageSerializer().save(self.payload, str(self.admin.id)) + self.assertFalse(data['is_active']) + self.assertTrue(UserActivationApplication.objects.filter(user_id=data['id'], status='PENDING').exists()) + + def test_anonymous_configuration_does_not_import_django_auth(self, _registry): + self.assertIsNone(settings.REST_FRAMEWORK.get('UNAUTHENTICATED_USER', 'missing')) + + def test_false_string_does_not_require_approval(self, _registry): + data = UserManageSerializer().save({**self.payload, 'pending_activation': 'false'}, str(self.admin.id)) + self.assertTrue(data['is_active']) + + def test_pending_user_hidden_from_management_list_and_page(self, _registry): + RegisterSerializer.register(self.payload) + query = UserManageSerializer.Query(data={'username': self.payload['username']}) + self.assertEqual(query.list(), []) + page = query.page(1, 20, str(self.admin.id)) + self.assertEqual(page['total'], 0) + self.assertEqual(page['records'], []) + self.assertEqual(ActivationSerializer.page(1, 20)['total'], 1) + + def test_approved_user_appears_enabled_in_management(self, _registry): + RegisterSerializer.register(self.payload) + application = UserActivationApplication.objects.get() + ActivationSerializer.approve(str(application.id), str(self.admin.id)) + page = UserManageSerializer.Query(data={'username': self.payload['username']}).page(1, 20, str(self.admin.id)) + self.assertEqual(page['total'], 1) + self.assertTrue(page['records'][0]['is_active']) + self.assertEqual(ActivationSerializer.page(1, 20)['total'], 0) + + def test_normal_disabled_account_remains_in_management(self, _registry): + data = UserManageSerializer().save({**self.payload, 'pending_activation': False}, str(self.admin.id)) + User.objects.filter(id=data['id']).update(is_active=False) + page = UserManageSerializer.Query(data={'username': self.payload['username'], 'is_active': False}).page(1, 20, str(self.admin.id)) + self.assertEqual(page['total'], 1) + self.assertFalse(page['records'][0]['is_active']) + + def test_rejection_preserves_account_and_allows_resubmission(self, _registry): + RegisterSerializer.register(self.payload) + application = UserActivationApplication.objects.get() + ActivationSerializer.reject(str(application.id), str(self.admin.id), '资料不符') + application.refresh_from_db() + self.assertEqual(application.reject_reason, '资料不符') + self.assertEqual(ActivationSerializer.page(1, 20)['total'], 0) + self.assertEqual(ActivationSerializer.page(1, 20, status=ActivationStatus.REJECTED)['total'], 1) + page = UserManageSerializer.Query(data={'username': self.payload['username']}).page(1, 20, str(self.admin.id)) + self.assertEqual(page['total'], 0) + self.assertEqual(page['records'], []) + user_id = application.user_id + result = RegisterSerializer.register({**self.payload, 'nick_name': '重新提交姓名', + 'phone': '13800138009', 'email': 'reapply@example.com'}) + self.assertTrue(result['re_register']) + application.refresh_from_db() + self.assertEqual(application.user_id, user_id) + self.assertEqual(application.status, ActivationStatus.PENDING) + self.assertEqual(application.reject_reason, '') + self.assertIsNone(application.audit_time) + self.assertIsNone(application.audit_by_id) + user = User.objects.get(id=user_id) + self.assertFalse(user.is_active) + self.assertEqual(user.nick_name, '重新提交姓名') + self.assertEqual(user.phone, '13800138009') + self.assertEqual(user.email, 'reapply@example.com') + self.assertEqual(application.email, user.email) + self.assertEqual(UserActivationApplication.objects.count(), 1) + + + def test_activation_audit_keeps_pending_and_rejected_with_phone(self, _registry): + for index, status in enumerate((ActivationStatus.PENDING, ActivationStatus.REJECTED, ActivationStatus.APPROVED)): + user = User.objects.create(username=f'audit_{index}', nick_name=f'审核{index}', role='USER', + phone=f'1380013800{index}', is_active=status == ActivationStatus.APPROVED) + UserActivationApplication.objects.create(user=user, status=status, + reject_reason='资料不符' if status == ActivationStatus.REJECTED else '') + page = ActivationSerializer.page(1, 20, status='UNAPPROVED') + self.assertEqual(page['total'], 2) + self.assertEqual({row['status'] for row in page['records']}, {'PENDING', 'REJECTED'}) + self.assertEqual({row['phone'] for row in page['records']}, {'13800138000', '13800138001'}) + rejected = next(row for row in page['records'] if row['status'] == 'REJECTED') + self.assertEqual(rejected['reject_reason'], '资料不符') + filtered = ActivationSerializer.page(1, 20, keyword='13800138001', status='UNAPPROVED') + self.assertEqual(filtered['total'], 1) + self.assertEqual(filtered['records'][0]['status'], 'REJECTED') + + def test_activation_audit_returns_empty_phone_when_not_provided(self, _registry): + RegisterSerializer.register(self.payload) + page = ActivationSerializer.page(1, 20, status='UNAPPROVED') + self.assertEqual(page['records'][0]['phone'], '') + + def test_profile_edit_cannot_change_roles(self, _registry): + user = User.objects.create(username='profile_user', nick_name='资料用户', role='USER') + for changes in ({'role': 'ADMIN'}, {'role_setting': [{'role_id': 'ADMIN'}]}): + with self.subTest(changes=changes): + with self.assertRaises(AppApiException): + UserManageSerializer.Operate(data={'id': str(user.id)}).edit(changes, str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.role, 'USER') + + def test_profile_edit_keeps_existing_role_assignment(self, _registry): + user = User.objects.create(username='profile_admin', nick_name='资料管理员', role='ADMIN') + assignment = UserRoleAssignment.objects.create(user=user, role_id='ADMIN', workspace_id='default') + UserManageSerializer.Operate(data={'id': str(user.id)}).edit({'nick_name': '更新姓名'}, str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.nick_name, '更新姓名') + self.assertEqual(user.role, 'ADMIN') + self.assertTrue(UserRoleAssignment.objects.filter(id=assignment.id).exists()) + + def test_role_choices_are_user_and_admin(self, _registry): + detail = RoleAssignmentSerializer.detail(str(self.admin.id)) + self.assertEqual(detail['available_roles'], [{'id': 'USER', 'name': '普通用户'}, {'id': 'ADMIN', 'name': '管理员'}]) + self.assertNotIn('workspaces', detail) + + def test_dedicated_role_assignment_updates_role_and_invalidates_cache(self, _registry): + user = User.objects.create(username='role_user', nick_name='授权用户', role='USER') + for version in (Cache_Version.PERMISSION_LIST, Cache_Version.ROLE_LIST, Cache_Version.USER): + cache.set(version.get_key(user_id=user.id), ['old'], version=version.get_version()) + RoleAssignmentSerializer.save(str(user.id), [{'role_id': 'ADMIN'}], str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.role, 'ADMIN') + assignment = UserRoleAssignment.objects.get(user=user) + self.assertEqual(assignment.role_id, 'ADMIN') + self.assertEqual(assignment.granted_by_id, self.admin.id) + self.assertEqual(assignment.workspace_id, 'default') + for version in (Cache_Version.PERMISSION_LIST, Cache_Version.ROLE_LIST, Cache_Version.USER): + self.assertIsNone(cache.get(version.get_key(user_id=user.id), version=version.get_version())) + RoleAssignmentSerializer.save(str(user.id), [{'role_id': 'USER'}], str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.role, 'USER') + self.assertEqual(UserRoleAssignment.objects.filter(user=user).count(), 1) + + def test_invalid_role_assignment_leaves_existing_role_unchanged(self, _registry): + user = User.objects.create(username='invalid_role', nick_name='角色校验', role='USER') + assignment = UserRoleAssignment.objects.create(user=user, role_id='USER', workspace_id='default') + for roles in ([{'role_id': 'WORKSPACE_MANAGE'}], [{'role_id': 'ADMIN'}, {'role_id': 'USER'}], + [{'role_id': 'ADMIN', 'workspace_ids': ['other']}], []): + with self.subTest(roles=roles): + with self.assertRaises(AppApiException): + RoleAssignmentSerializer.save(str(user.id), roles, str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.role, 'USER') + self.assertTrue(UserRoleAssignment.objects.filter(id=assignment.id).exists()) + + def test_builtin_admin_role_cannot_be_downgraded(self, _registry): + user = User.objects.create(id='f0dd8f71-e4ee-11ee-8c84-a8a1595801ab', username='builtin', nick_name='内置管理员', role='ADMIN') + with self.assertRaises(AppApiException): + RoleAssignmentSerializer.save(str(user.id), [{'role_id': 'USER'}], str(self.admin.id)) + user.refresh_from_db() + self.assertEqual(user.role, 'ADMIN') + + def test_approved_and_pending_cannot_reregister(self, _registry): + RegisterSerializer.register(self.payload) + app = UserActivationApplication.objects.get() + for status in (ActivationStatus.PENDING, ActivationStatus.APPROVED): + app.status = status + app.save() + with self.subTest(status=status), self.assertRaises(AppApiException): + RegisterSerializer.register({**self.payload, 'phone': '13800138008'}) + self.assertEqual(User.objects.get(username=self.payload['username']).phone, '') + + def test_rejected_requires_original_credentials(self, _registry): + RegisterSerializer.register(self.payload) + app = UserActivationApplication.objects.get() + ActivationSerializer.reject(str(app.id), str(self.admin.id), '资料不符') + for changed in ({'password': 'Other@123', 're_password': 'Other@123'}, {'username': 'other_name'}): + with self.subTest(changed=changed), self.assertRaises(AppApiException): + RegisterSerializer.register({**self.payload, **changed}) + app.refresh_from_db() + self.assertEqual(app.status, ActivationStatus.REJECTED) + self.assertEqual(app.reject_reason, '资料不符') + + def test_reapply_does_not_overwrite_another_account(self, _registry): + RegisterSerializer.register(self.payload) + app = UserActivationApplication.objects.get() + ActivationSerializer.reject(str(app.id), str(self.admin.id), '资料不符') + other = User.objects.create(username='other', nick_name='其他姓名', email='other@example.com', role='USER') + for changed in ({'nick_name': other.nick_name}, {'email': other.email}): + with self.subTest(changed=changed), self.assertRaises(AppApiException): + RegisterSerializer.register({**self.payload, **changed}) + app.refresh_from_db() + self.assertEqual(app.status, ActivationStatus.REJECTED) + + @patch.object(LoginSerializer, 'get_auth_setting', return_value={'max_attempts': -1}) + def test_rejected_login_includes_reason_only_with_correct_password(self, _settings, _registry): + RegisterSerializer.register(self.payload) + app = UserActivationApplication.objects.get() + ActivationSerializer.reject(str(app.id), str(self.admin.id), '手机号与申请资料不一致') + with self.assertRaises(AppApiException) as error: + LoginSerializer.login({'username': self.payload['username'], 'password': self.payload['password']}) + self.assertIn('手机号与申请资料不一致', str(error.exception)) + with self.assertRaises(AppApiException) as error: + LoginSerializer.login({'username': self.payload['username'], 'password': 'Wrong@123'}) + self.assertNotIn('手机号与申请资料不一致', str(error.exception)) + + def test_audit_notices_are_scheduled_after_commit(self, _registry): + RegisterSerializer.register(self.payload) + app = UserActivationApplication.objects.get() + with patch('users.serializers.user_manage_ext.send_activation_notice') as notice: + with self.captureOnCommitCallbacks(execute=True): + ActivationSerializer.reject(str(app.id), str(self.admin.id), '资料不符') + notice.assert_not_called() + self.assertEqual(notice.call_args.args[1:], (False, '资料不符')) + RegisterSerializer.register(self.payload) + with patch('users.serializers.user_manage_ext.send_activation_notice') as notice: + with self.captureOnCommitCallbacks(execute=True): + ActivationSerializer.approve(str(app.id), str(self.admin.id)) + notice.assert_not_called() + self.assertTrue(notice.call_args.args[1]) + + def test_notice_without_email_configuration_does_not_send(self, _registry): + with patch('users.serializers.user_manage_ext.SystemSetting.objects') as setting, \ + patch('users.serializers.user_manage_ext.send_mail') as mail: + setting.filter.return_value.first.return_value = None + self.assertFalse(send_activation_notice(self.admin, True)) + mail.assert_not_called() + + def test_notice_contains_result_and_reason(self, _registry): + from types import SimpleNamespace + with patch('users.serializers.user_manage_ext.SystemSetting.objects') as setting, \ + patch('users.serializers.user_manage_ext.EmailBackend'), \ + patch('users.serializers.user_manage_ext.send_mail', return_value=1) as mail: + setting.filter.return_value.first.return_value = SimpleNamespace(meta={ + 'email_host': 'smtp.example.invalid', 'email_port': 25, 'from_email': 'sender@example.com'}) + self.assertTrue(send_activation_notice(self.admin, False, '手机号不符')) + self.assertIn('未通过', mail.call_args.args[0]) + self.assertIn('手机号不符', mail.call_args.args[1]) + self.assertEqual(mail.call_args.kwargs['recipient_list'], [self.admin.email]) + self.assertTrue(send_activation_notice(self.admin, True)) + self.assertIn('审核通过', mail.call_args.args[0]) + + def test_mail_failure_does_not_change_audit_outcome(self, _registry): + from types import SimpleNamespace + with patch('users.serializers.user_manage_ext.SystemSetting.objects') as setting, \ + patch('users.serializers.user_manage_ext.EmailBackend'), \ + patch('users.serializers.user_manage_ext.send_mail', side_effect=RuntimeError('mock SMTP error')): + setting.filter.return_value.first.return_value = SimpleNamespace(meta={'email_host': 'smtp.example.invalid'}) + self.assertFalse(send_activation_notice(self.admin, True)) + + def test_resource_user_lists_exclude_unapproved_and_disabled(self, _registry): + allowed = User.objects.create(username='allowed', nick_name='已通过', role='USER') + UserActivationApplication.objects.create(user=allowed, status=ActivationStatus.APPROVED) + direct = User.objects.create(username='direct', nick_name='直接激活', role='USER') + for status in (ActivationStatus.PENDING, ActivationStatus.REJECTED): + user = User.objects.create(username=status, nick_name=status, role='USER', is_active=True) + UserActivationApplication.objects.create(user=user, status=status) + User.objects.create(username='disabled', nick_name='停用', role='USER', is_active=False) + members = UserManageSerializer().get_user_members('default') + self.assertEqual({str(u['id']) for u in members}, {str(allowed.id), str(direct.id)}) + for entries in (UserManageSerializer().get_user_list('default'), UserManageSerializer().get_all_user_list()): + self.assertEqual({str(u['id']) for u in entries}, {str(allowed.id), str(direct.id), str(self.admin.id)}) + + def test_resource_authorization_cannot_use_hidden_user_id(self, _registry): + RegisterSerializer.register(self.payload) + user = User.objects.get(username=self.payload['username']) + for kind in ('APPLICATION', 'KNOWLEDGE', 'TOOL', 'MODEL'): + with self.subTest(kind=kind), self.assertRaises(AppApiException): + UserResourcePermissionSerializer(data={'workspace_id': 'default', 'user_id': str(user.id), + 'auth_target_type': kind}).is_valid(raise_exception=True) + with self.subTest(kind=kind), self.assertRaises(AppApiException): + ResourceUserPermissionSerializer(data={'workspace_id': 'default', 'target': 'test-target', + 'auth_target_type': kind}).edit([{'user_id': str(user.id), 'permission': 'VIEW'}]) + + def test_direct_activation_schedules_notice(self, _registry): + with patch('users.serializers.user_manage_ext.send_activation_notice') as notice: + with self.captureOnCommitCallbacks(execute=True): + UserManageSerializer().save({**self.payload, 'pending_activation': False}, str(self.admin.id)) + notice.assert_not_called() + self.assertTrue(notice.call_args.args[1]) + + def test_resource_user_sql_filters_all_four_modules(self, _registry): + from django.db import connection + from common.db.search import compiler_queryset + allowed = User.objects.create(username='allowed_sql', nick_name='SQL已通过', role='USER') + UserActivationApplication.objects.create(user=allowed, status=ActivationStatus.APPROVED) + for status in (ActivationStatus.PENDING, ActivationStatus.REJECTED): + user = User.objects.create(username='sql_'+status, nick_name='SQL'+status, role='USER') + UserActivationApplication.objects.create(user=user, status=status) + User.objects.create(username='disabled_sql', nick_name='SQL停用', role='USER', is_active=False) + for kind in ('APPLICATION', 'KNOWLEDGE', 'TOOL', 'MODEL'): + with self.subTest(kind=kind): + serializer = ResourceUserPermissionSerializer(data={ + 'workspace_id': 'default', 'target': 'test', 'auth_target_type': kind}) + serializer.is_valid(raise_exception=True) + query = serializer.get_queryset({'permission': []}, False)['user_query_set'] + clause, params = compiler_queryset(query) + with connection.cursor() as cursor: + cursor.execute('SELECT u.id FROM "user" u ' + clause, params) + ids = {str(row[0]).replace('-', '') for row in cursor.fetchall()} + self.assertEqual(ids, {str(allowed.id).replace('-', '')}) diff --git a/apps/users/urls.py b/apps/users/urls.py index 6f8cd8565..1bf4d6ba5 100644 --- a/apps/users/urls.py +++ b/apps/users/urls.py @@ -6,6 +6,7 @@ app_name = "user" # @formatter:off urlpatterns = [ path('user/login', views.LoginView.as_view(), name='login'), + path('user/register', views.RegisterView.as_view(), name='register'), path('user/profile', views.UserProfileView.as_view(), name="user_profile"), path('user/captcha', views.CaptchaView.as_view(), name='captcha'), path('user/test', views.TestPermissionsUserView.as_view(), name="test"), @@ -21,6 +22,12 @@ urlpatterns = [ path('workspace//user_member',views.WorkspaceUserMemberView.as_view(), name="test_workspace_id_permission"), path('workspace//user/profile', views.TestWorkspacePermissionUserView.as_view(), name="test_workspace_id_permission"), path("user_manage", views.UserManage.as_view(), name="user_manage"), + path("user_manage/activation/pending", views.UserActivationPendingView.as_view(), name="user_activation_pending"), + path("user_manage/activation//approve", views.UserActivationApproveView.as_view(), + name="user_activation_approve"), + path("user_manage/activation//reject", views.UserActivationRejectView.as_view(), + name="user_activation_reject"), + path("user_manage//roles", views.UserRoleAssignmentView.as_view(), name="user_role_assignment"), path("user_manage/batch_delete", views.UserManage.BatchDelete.as_view()), path("user_manage/password", views.UserManage.Password.as_view()), path("user_manage/", views.UserManage.Operate.as_view(), name="user_manage_operate"), diff --git a/apps/users/views/__init__.py b/apps/users/views/__init__.py index 9ef4e79ce..47aaf015d 100644 --- a/apps/users/views/__init__.py +++ b/apps/users/views/__init__.py @@ -8,3 +8,5 @@ """ from .login import * from .user import * +from .user_manage_ext import * +from .register import RegisterView diff --git a/main.py b/main.py index f738764fb..bf17604fb 100644 --- a/main.py +++ b/main.py @@ -79,7 +79,7 @@ def dev(): if __name__ == '__main__': - os.environ['HF_HOME'] = '/opt/maxkb-app/model/base' + os.environ.setdefault('HF_HOME', '/opt/maxkb-app/model/base') parser = argparse.ArgumentParser( description=""" qabot service control tools; diff --git a/ui/src/api/knowledge/knowledge.ts b/ui/src/api/knowledge/knowledge.ts index 572ae237a..56dd4f66c 100644 --- a/ui/src/api/knowledge/knowledge.ts +++ b/ui/src/api/knowledge/knowledge.ts @@ -304,6 +304,91 @@ const delMulTag: (knowledge_id: string, tags: any, loading?: Ref) => Pr return put(`${prefix.value}/${knowledge_id}/tags/batch_delete`, tags, null, loading) } +/** + * 独立重建知识库索引 + */ +const postRebuildKnowledgeIndex: ( + knowledge_id: string, + loading?: Ref, +) => Promise> = (knowledge_id, loading) => { + return post(`${prefix.value}/${knowledge_id}/rebuild`, undefined, undefined, loading) +} + +/** + * 重建任务列表 + */ +const getRebuildTasks: ( + knowledge_id: string, + page: pageRequest, + loading?: Ref, +) => Promise> = (knowledge_id, page, loading) => { + return get(`${prefix.value}/${knowledge_id}/rebuild`, page, loading) +} + +/** + * 重建任务详情 + */ +const getRebuildTask: ( + knowledge_id: string, + task_id: string, + loading?: Ref, +) => Promise> = (knowledge_id, task_id, loading) => { + return get(`${prefix.value}/${knowledge_id}/rebuild/${task_id}`, undefined, loading) +} + +/** + * 重试重建任务 + */ +const postRetryRebuildTask: ( + knowledge_id: string, + task_id: string, + loading?: Ref, +) => Promise> = (knowledge_id, task_id, loading) => { + return post(`${prefix.value}/${knowledge_id}/rebuild/${task_id}/retry`, undefined, undefined, loading) +} + +/** + * 知识库业务元数据 + */ +const getKnowledgeMetadata: ( + knowledge_id: string, + loading?: Ref, +) => Promise> = (knowledge_id, loading) => { + return get(`${prefix.value}/${knowledge_id}/metadata`, undefined, loading) +} + +/** + * 修改知识库业务元数据 + */ +const putKnowledgeMetadata: ( + knowledge_id: string, + data: Record, + loading?: Ref, +) => Promise> = (knowledge_id, data, loading) => { + return put(`${prefix.value}/${knowledge_id}/metadata`, data, undefined, loading) +} + +/** + * 知识库分段规则 + */ +const getKnowledgeSegmentation: ( + knowledge_id: string, + loading?: Ref, +) => Promise> = (knowledge_id, loading) => { + return get(`${prefix.value}/${knowledge_id}/segmentation`, undefined, loading) +} + +/** + * 修改知识库分段规则 + */ +const putKnowledgeSegmentation: ( + knowledge_id: string, + data: { chunk_size: number; chunk_overlap: number }, + loading?: Ref, +) => Promise> = (knowledge_id, data, loading) => { + return put(`${prefix.value}/${knowledge_id}/segmentation`, data, undefined, loading) +} + export default { getKnowledgeList, getKnowledgeListPage, @@ -326,5 +411,13 @@ export default { postTags, putTag, delTag, - delMulTag + delMulTag, + postRebuildKnowledgeIndex, + getRebuildTasks, + getRebuildTask, + postRetryRebuildTask, + getKnowledgeMetadata, + putKnowledgeMetadata, + getKnowledgeSegmentation, + putKnowledgeSegmentation } diff --git a/ui/src/api/system/user-manage.ts b/ui/src/api/system/user-manage.ts index efc329143..fcb5120ca 100644 --- a/ui/src/api/system/user-manage.ts +++ b/ui/src/api/system/user-manage.ts @@ -91,6 +91,60 @@ const getValid: ( } +/** + * 待激活注册申请分页 + */ +const getActivationPending: ( + params: any, + loading?: Ref, +) => Promise> = (params, loading) => { + return get(`${prefix}/activation/pending`, params, loading) +} + +/** + * 通过待激活申请 + */ +const approveActivation: ( + application_id: string, + data: any, + loading?: Ref, +) => Promise> = (application_id, data, loading) => { + return post(`${prefix}/activation/${application_id}/approve`, data, undefined, loading) +} + +/** + * 拒绝待激活申请 + */ +const rejectActivation: ( + application_id: string, + data: any, + loading?: Ref, +) => Promise> = (application_id, data, loading) => { + return post(`${prefix}/activation/${application_id}/reject`, data, undefined, loading) +} + +/** + * 查看用户角色授权 + */ +const getUserRoles: (user_id: string, loading?: Ref) => Promise> = ( + user_id, + loading, +) => { + return get(`${prefix}/${user_id}/roles`, undefined, loading) +} + +/** + * 分配用户角色 + */ +const putUserRoles: ( + user_id: string, + data: any, + loading?: Ref, +) => Promise> = (user_id, data, loading) => { + return put(`${prefix}/${user_id}/roles`, data, undefined, loading) +} + + export default { getUserManage, putUserManage, @@ -99,5 +153,9 @@ export default { putUserManagePassword, getSystemDefaultPassword, getValid, - + getActivationPending, + approveActivation, + rejectActivation, + getUserRoles, + putUserRoles, } diff --git a/ui/src/api/type/user.ts b/ui/src/api/type/user.ts index 094b3d31b..3c48ecf80 100644 --- a/ui/src/api/type/user.ts +++ b/ui/src/api/type/user.ts @@ -44,26 +44,11 @@ interface LoginRequest { } interface RegisterRequest { - /** - * 用户名 - */ username: string - /** - * 密码 - */ - password: string - /** - * 确定密码 - */ - re_password: string - /** - * 邮箱 - */ + nick_name: string email: string - /** - * 验证码 - */ - code: string + phone: string + encryptedData: string } interface CheckCodeRequest { diff --git a/ui/src/api/user/user.ts b/ui/src/api/user/user.ts index 2f2a5de38..72c9ccff2 100644 --- a/ui/src/api/user/user.ts +++ b/ui/src/api/user/user.ts @@ -1,7 +1,11 @@ import { Result } from '@/request/Result' import { get, post } from '@/request/index' -import type { User, ResetPasswordRequest, CheckCodeRequest } from '@/api/type/user' +import type { User, ResetPasswordRequest, CheckCodeRequest, RegisterRequest } from '@/api/type/user' import type { Ref } from 'vue' + +const register = (request: RegisterRequest, loading?: Ref): Promise> => { + return post('/user/register', request, undefined, loading) +} /** * 获取用户基本信息 * @param loading 接口加载器 @@ -90,6 +94,7 @@ const resetCurrentPassword: ( } export default { + register, getUserProfile, getProfile, getUserList, diff --git a/ui/src/components/app-table/index.vue b/ui/src/components/app-table/index.vue index 72c8748ae..0a244ed35 100644 --- a/ui/src/components/app-table/index.vue +++ b/ui/src/components/app-table/index.vue @@ -47,7 +47,7 @@ + diff --git a/ui/vite.config.ts b/ui/vite.config.ts index 3c1f441b4..a0c3a6876 100644 --- a/ui/vite.config.ts +++ b/ui/vite.config.ts @@ -1,5 +1,5 @@ import { fileURLToPath, URL } from 'node:url' -import type { ProxyOptions } from 'vite' +import type { ProxyOptions, ViteDevServer } from 'vite' import { defineConfig, loadEnv } from 'vite' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' @@ -34,39 +34,40 @@ const renameHtmlPlugin = (outDir: string, entry: string) => { export default defineConfig((conf: any) => { const mode = conf.mode const ENV = loadEnv(mode, envDir) + const backendUrl = process.env.VITE_BACKEND_URL || ENV.VITE_BACKEND_URL || 'http://localhost:8080' const proxyConf: Record = {} proxyConf['/admin/api'] = { - target: 'http://localhost:8080', + target: backendUrl, changeOrigin: true, } proxyConf['/chat/api'] = { - target: 'http://localhost:8080', + target: backendUrl, changeOrigin: true, } proxyConf['/doc'] = { - target: 'http://localhost:8080', + target: backendUrl, changeOrigin: true, rewrite: (path: string) => path.replace(ENV.VITE_BASE_PATH, '/'), } proxyConf['/schema'] = { - target: 'http://localhost:8080', + target: backendUrl, changeOrigin: true, rewrite: (path: string) => path.replace(ENV.VITE_BASE_PATH, '/'), } proxyConf['/static'] = { - target: 'http://localhost:8080', + target: backendUrl, changeOrigin: true, rewrite: (path: string) => path.replace(ENV.VITE_BASE_PATH, '/'), } // 前端静态资源转发到本身 proxyConf[`^${ENV.VITE_BASE_PATH}.+\/oss\/file\/.*$`] = { - target: `http://localhost:8080`, + target: backendUrl, changeOrigin: true, } // 前端静态资源转发到本身 proxyConf[`^${ENV.VITE_BASE_PATH}oss\/file\/.*$`] = { - target: `http://localhost:8080`, + target: backendUrl, changeOrigin: true, } // 前端静态资源转发到本身 @@ -85,6 +86,24 @@ export default defineConfig((conf: any) => { vue(), vueJsx(), DefineOptions(), + { + name: 'dev-app-entry', + configureServer(server: ViteDevServer) { + server.middlewares.use((req, _res, next) => { + const pathname = (req.url || '/').split('?')[0] + const prefix = ENV.VITE_BASE_PATH.replace(/\/$/, '') + if ( + req.headers.accept?.includes('text/html') && + (pathname === prefix || pathname.startsWith(`${prefix}/`)) && + !pathname.startsWith(`${prefix}/api/`) && + !pathname.includes('.') + ) { + req.url = `/${ENV.VITE_ENTRY}` + } + next() + }) + }, + }, createHtmlPlugin({ template: ENV.VITE_ENTRY }), renameHtmlPlugin(`dist${ENV.VITE_BASE_PATH}`, ENV.VITE_ENTRY), ],