diff --git a/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py b/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py index 8a80b0d..feb9613 100644 --- a/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py +++ b/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py @@ -296,8 +296,8 @@ def union_make(pdf_info_dict: list, def get_title_level(block): title_level = block.get('level', 1) - if title_level > 4: - title_level = 4 + if title_level > 6: + title_level = 6 elif title_level < 1: title_level = 0 return title_level @@ -311,4 +311,4 @@ def escape_special_markdown_char(content): for char in special_chars: content = content.replace(char, "\\" + char) - return content \ No newline at end of file + return content diff --git a/mineru/backend/vlm/vlm_middle_json_mkcontent.py b/mineru/backend/vlm/vlm_middle_json_mkcontent.py index c081cd6..4c0ef9c 100644 --- a/mineru/backend/vlm/vlm_middle_json_mkcontent.py +++ b/mineru/backend/vlm/vlm_middle_json_mkcontent.py @@ -651,8 +651,8 @@ def union_make(pdf_info_dict: list, def get_title_level(block): title_level = block.get('level', 1) - if title_level > 4: - title_level = 4 + if title_level > 6: + title_level = 6 elif title_level < 1: title_level = 0 return title_level diff --git a/mineru/utils/llm_aided.py b/mineru/utils/llm_aided.py index 03e669d..e8f7a53 100644 --- a/mineru/utils/llm_aided.py +++ b/mineru/utils/llm_aided.py @@ -54,7 +54,7 @@ def llm_aided_title(page_info_list, title_aided_config): - 根据标题内容的语义为每个标题元素添加适当的层次结构 - 行高较大的标题一般是更高级别的标题 - 标题从前至后的层级必须是连续的,不能跳过层级 - - 标题层级最多为4级,不要添加过多的层级 + - 标题层级最多为6级,不要添加过多的层级 - 优化后的标题只保留代表该标题的层级的整数,不要保留其他信息 5. 合理性检查与微调: diff --git a/tests/unittest/test_heading_levels.py b/tests/unittest/test_heading_levels.py new file mode 100644 index 0000000..e84d4ae --- /dev/null +++ b/tests/unittest/test_heading_levels.py @@ -0,0 +1,54 @@ +"""Heading post-processing checks; no OCR, model inference or external requests.""" +import copy +import json +from types import SimpleNamespace +import unittest +from unittest.mock import MagicMock, patch + +from mineru.backend.pipeline import pipeline_middle_json_mkcontent as pipeline +from mineru.backend.vlm import vlm_middle_json_mkcontent as vlm +from mineru.utils.enum_class import MakeMode +from mineru.utils import llm_aided + + +class HeadingLevelsTests(unittest.TestCase): + def blocks(self): + return [{'type': 'title', 'level': level, 'bbox': [0, level * 10, 100, level * 10 + 8], + 'line_avg_height': 20 - level, + 'lines': [{'bbox': [0, 0, 100, 8], 'spans': [{'type': 'text', 'content': f'Heading{level}'}]}]} + for level in range(1, 7)] + + def pages(self): + return [{'page_idx': 0, 'page_size': [600, 800], 'para_blocks': self.blocks(), 'discarded_blocks': []}] + + def test_both_output_backends_preserve_six_headings_in_markdown_and_metadata(self): + for converter in (pipeline, vlm): + with self.subTest(converter=converter.__name__), patch.object(converter, 'detect_lang', return_value='en'): + markdown = converter.union_make(copy.deepcopy(self.pages()), MakeMode.MM_MD) + self.assertEqual(markdown.split('\n\n'), [f'{"#" * i} Heading{i}' for i in range(1, 7)]) + content = converter.union_make(copy.deepcopy(self.pages()), MakeMode.CONTENT_LIST) + self.assertEqual([item['text_level'] for item in content], list(range(1, 7))) + content_v2 = vlm.union_make(self.pages(), MakeMode.CONTENT_LIST_V2) + self.assertEqual([item['content']['level'] for page in content_v2 for item in page], list(range(1, 7))) + + def test_out_of_range_levels_still_produce_valid_markdown(self): + for converter in (pipeline, vlm): + self.assertEqual(converter.get_title_level({}), 1) + for value, expected in [(-1, 0), (0, 0), (5, 5), (6, 6), (7, 6), (99, 6)]: + self.assertEqual(converter.get_title_level({'level': value}), expected) + + def test_assisted_title_prompt_allows_six_levels(self): + client = MagicMock() + client.chat.completions.create.return_value = [SimpleNamespace(choices=[SimpleNamespace( + delta=SimpleNamespace(content=json.dumps({str(i): i + 1 for i in range(6)})))])] + pages = self.pages() + with patch.object(llm_aided, 'OpenAI', return_value=client), patch.object(pipeline, 'detect_lang', return_value='en'): + llm_aided.llm_aided_title(pages, {'api_key': 'unused-test-value', 'base_url': 'https://example.invalid', 'model': 'test'}) + instruction = client.chat.completions.create.call_args.kwargs['messages'][0]['content'] + self.assertIn('标题层级最多为6级', instruction) + self.assertNotIn('标题层级最多为4级', instruction) + self.assertEqual([block['level'] for block in pages[0]['para_blocks']], list(range(1, 7))) + + +if __name__ == '__main__': + unittest.main(verbosity=2)