"""Conservative outline repair. Rules/models change heading markers, never body text. All line numbers are zero based. Stable IDs and explicit manual locks survive repair and artifact revisions. Layout scores are heuristics, not probabilities. """ import copy import hashlib import json import math import re from collections import Counter, defaultdict from statistics import median MAX_HEADINGS = 6000 HEADING = re.compile(r'^( {0,3})(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$') DECIMAL = re.compile(r'^(\d{1,3}(?:\.\d{1,3}){0,5})(?:[.、.])?[ \t]+(\S.*)$') CHINESE = [('chapter', re.compile(r'^第[一二三四五六七八九十百千零〇\d]+[章节篇部]\s*\S*')), ('chinese', re.compile(r'^[一二三四五六七八九十百]+[、..]\s*\S+')), ('paren_chinese', re.compile(r'^[((][一二三四五六七八九十百]+[))]\s*\S+')), ('paren_digit', re.compile(r'^[((]\d{1,3}[))]\s*\S+'))] def label(text): return re.sub(r'\s+', ' ', re.sub(r'[*_`]', '', text)).strip() def _short(text): return 0 < len(text) <= 120 and not re.search(r'[。!?;;!?]|https?://|\|', text) def _number(text): match = DECIMAL.match(text) if not match or not _short(text) or re.match(r'^(?:19|20)\d{2}\b', text): return None prefix = match[1] if len(prefix.split('.')) == 4 and all(int(x) <= 255 for x in prefix.split('.')) and ( int(prefix.split('.')[0]) >= 10 or re.search(r'\b(?:IP|gateway|address|DNS)\b|地址|网关|掩码', match[2], re.I)): # An IP address is not a chapter number without independent evidence. return None if re.match(r'^(?:版本|version|v\d|\d+(?:\.\d+)+\s*(?:版|版本))', text, re.I): return None return prefix def _protected_lines(markdown): from markdown_it import MarkdownIt tokens = MarkdownIt().enable('table').parse(markdown) if len(tokens) > 60000: raise ValueError('文档结构过于复杂,请拆分文档') excluded, lists = set(), set() depth = 0 for token in tokens: if token.type in ('bullet_list_open', 'ordered_list_open'): depth += 1 if token.map and token.type in ('fence', 'code_block', 'table_open', 'html_block'): excluded.update(range(*token.map)) if token.map and depth: lists.update(range(*token.map)) if token.type in ('bullet_list_close', 'ordered_list_close'): depth -= 1 return excluded, lists def _evidence(source): """Normalize native evidence and MinerU middle JSON without exposing provider config.""" evidence = [] for item in source.get('blocks', []) if isinstance(source, dict) else []: if isinstance(item, dict) and isinstance(item.get('text'), str): evidence.append(dict(item)) middle = source.get('middle_json', {}) if isinstance(source, dict) else {} for page in middle.get('pdf_info', []) if isinstance(middle, dict) else []: if not isinstance(page, dict): continue size = page.get('page_size') or [0, 0] for block in (page.get('para_blocks') or []) + (page.get('discarded_blocks') or []): if not isinstance(block, dict): continue text = ''.join(str(s.get('content', '')) for line in block.get('lines', []) for s in line.get('spans', []) if s.get('type') == 'text') evidence.append({'text': text, 'page': page.get('page_idx', 0), 'bbox': block.get('bbox'), 'page_height': size[1] if len(size) > 1 else 0, 'kind': block.get('type'), 'height': block.get('line_avg_height') or median([ abs(line['bbox'][3] - line['bbox'][1]) for line in block.get('lines', []) if isinstance(line.get('bbox'), list) and len(line['bbox']) == 4] or [0])}) for item in evidence: for key in ('font_size', 'height', 'page_height'): value = item.get(key) item[key] = value if type(value) in (int, float) and math.isfinite(value) and 0 <= value < 100000 else 0 box = item.get('bbox') if not isinstance(box, list) or len(box) != 4 or any(type(x) not in (int, float) or not math.isfinite(x) for x in box): item['bbox'] = None return evidence[:40000] def analyze(markdown, source=None, previous=None, repair=True): source, previous = source or {}, previous or {} if not isinstance(source, dict) or not isinstance(previous, dict): raise ValueError('标题结构数据无效') old_headings = previous.get('headings', []) if not isinstance(old_headings, list) or len(old_headings) > MAX_HEADINGS or any( not isinstance(h, dict) or not isinstance(h.get('id'), str) or len(h['id']) > 128 or not isinstance(h.get('text'), str) or type(h.get('level')) is not int or not 0 <= h['level'] <= 6 for h in old_headings): raise ValueError('标题候选数据无效') lines = markdown.splitlines(keepends=True) excluded, list_lines = _protected_lines(markdown) evidence = _evidence(source) if not evidence: evidence = _evidence({'blocks': [{**r, 'kind': 'title' if r.get('level') else 'text'} for r in old_headings]}) by_text = defaultdict(list) bookmarks = defaultdict(list) for item in source.get('bookmarks', []): if isinstance(item, dict) and type(item.get('level')) is int and 1 <= item['level'] <= 6: bookmarks[label(item.get('text', ''))].append(item) for block in evidence: by_text[label(block['text'])].append(block) old_by_text = defaultdict(list) for item in previous.get('headings', []): if isinstance(item, dict): old_by_text[item.get('text', '')].append(item) used_ids, counts, records = set(), Counter(), [] body_sizes = [b.get('font_size', 0) or b.get('height', 0) for b in evidence if b.get('kind') in ('text', 'paragraph') and (b.get('font_size', 0) or b.get('height', 0)) > 0] body_size = median(body_sizes) if body_sizes else 0 evidence_cursor = Counter() margins = defaultdict(set) for b in evidence: box, height = b.get('bbox'), b.get('page_height', 0) if isinstance(box, list) and len(box) == 4 and height and (box[3] <= height * .1 or box[1] >= height * .9): margins[label(b['text'])].add(b.get('page')) toc_lines = set() for pos, raw in enumerate(lines): if label(re.sub(r'^#{1,6}\s+', '', raw.strip())).lower() not in ('目录', 'contents', 'table of contents'): continue candidates = [] for index in range(pos + 1, min(len(lines), pos + 250)): if re.search(r'(?:\.{3,}|…{2,}|·{3,})\s*\d+\s*$', lines[index]): candidates.append(index) elif candidates and HEADING.match(lines[index].rstrip('\r\n')): break if len(candidates) >= 2: toc_lines.update(candidates) for index, raw in enumerate(lines): if index in excluded: continue line = raw.rstrip('\r\n') match = HEADING.match(line) text = label(match[3] if match else line.strip()) if not text or text.startswith(('![', '