"""Section-owned summaries: budgeted batches, local repair and scoped reuse. The model owns body bullets only. Titles, order, IDs and source ranges are owned by the server. No downloads, embeddings or additional services. """ import hashlib import json import re import threading import time from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from decimal import Decimal, InvalidOperation from system_manage.ai_hub_organize import split_markdown, token_bound from system_manage.summary_plan import ( language_matches, numeric_boundaries, output_estimate, source_language, ) VERSION = 'sections-v1' MARKER = re.compile(r'(?m)^#{1,6}\s+\[([a-zA-Z0-9_-]+)\]\s*$') NUMBER = re.compile(r'(?= fence[1]: fence = None current['body'] += line continue heading = None if fence else re.match(r'^(#{1,6})\s+(.+?)\s*#*\s*$', line.rstrip()) if heading: if current['heading'] or current['body'].strip(): current['end'] = i - 1 result.append(current) level = len(heading[1]) path = [(n, t) for n, t in path if n < level] + [(level, heading[2])] key = tuple(path) occurrences[key] += 1 current = {'heading': line.rstrip(), 'path': [t for _, t in path], 'id': 's' + digest([path, occurrences[key]])[:14], 'start': i, 'body_start': i + 1, 'body': ''} else: current['body'] += line if current['heading'] or current['body'].strip(): current['end'] = len(lines) result.append(current) for section in result: section.setdefault('id', 's' + digest(['preamble'])[:14]) return result def normalized_numbers(text): # OCR LaTeX often separates the digits: $4 0 0$ -> $400$. def compact_math(match): return re.sub(r'(?<=\d)\s+(?=\d)', '', match[0]) text = re.sub(r'\$\$[\s\S]*?\$\$|\$[^\n$]+\$', compact_math, text) text = re.sub(r'AIHUBIMAGE\s*\d+\s*TOKEN|https?://\S+', '', text) text = re.sub(r'(?m)^\s*(?:#{1,6}\s+|\d+[.)]\s+)', '', text) text = re.sub(r'(?<=\d)[–—-](?=\d)', '–', text) values = set() for match in NUMBER.finditer(text): raw = match[0].replace(',', '').replace(' ', '').rstrip('%') try: values.add(str(Decimal(raw).normalize())) except InvalidOperation: pass return values def facts_match(source, result, labels=''): """Deterministic guard, not a general semantic proof. Reject invented numbers and changed explicit conditions. Bind numeric sentences to distinctive source terms so exchanged subjects/units cannot pass simply because their numbers occur elsewhere in the document. """ if not normalized_numbers(result) <= normalized_numbers(source + '\n' + labels): return False, '新增或改变数字' expected, actual = numeric_boundaries(source), numeric_boundaries(result) if any(number in actual and not actual[number] <= operators for number, operators in expected.items()): return False, '数值边界条件改变或遗漏' if len(source) <= 512 and any(actual.get(number) != operators for number, operators in expected.items()): return False, '数值边界条件改变或遗漏' def quantities(text): units = {'days': 'day', 'day': 'day', '天': 'day', 'hours': 'hour', 'hour': 'hour', '小时': 'hour', 'years': 'year', 'year': 'year', '年': 'year', 'months': 'month', 'month': 'month', '月': 'month', 'weeks': 'week', 'week': 'week', '周': 'week', '℃': 'celsius', '°c': 'celsius', 'degrees celsius': 'celsius', 'degree celsius': 'celsius', 'degrees fahrenheit': 'fahrenheit', 'degree fahrenheit': 'fahrenheit', '°f': 'fahrenheit', '毫米': 'mm', '厘米': 'cm', '米': 'm'} pattern = r'(?= min(2, len(anchors)) and anchors: if not re.search(prohibition, target, re.I): return False, '否定或禁止条件改变' # Work locally within each numeric statement, including table rows. source_statements = re.split(r'(?<=[。;;!?])|(? input_limit: raise ValueError('章节路径或提示词超过输入预算,请调大输入上限') units.append(unit) offset += len(body) batches, current = [], [] for unit in units: candidate = current + [unit] if current and (count(envelope(candidate)) + count(guidance) + 512 > input_limit or sum(u['output_estimate'] for u in candidate) > output_limit * .85): batches.append(current) current = [] current.append(unit) if current: batches.append(current) if len(batches) > 128: raise ValueError('文档分块过多,请调大模型预算或拆分文档') if len(batches) > 1 and not options.get('auto_split', True): raise ValueError('文档超过输入或输出预算,请开启自动拆分') metrics = {'map_chunks': len(batches), 'sections': len(document), 'units': len(units), 'expected_calls': len(batches) + int(options.get('structure_mode') == 'reorganize' and bool(units)), 'model_calls': 0, 'reused_chunks': 0, 'concurrency': min(len(batches), options.get('concurrency', 1)) if options.get('parallel', True) else min(1, len(batches)), 'input_budget': input_limit, 'body_budget': body_budget, **settings, 'input_tokens_estimate': count(markdown), 'input_bytes': token_bound(markdown), 'estimated_output_tokens': sum(u['output_estimate'] for u in units), 'needs_merge': options.get('structure_mode') == 'reorganize', 'repair_calls': 0, 'fallback_sections': 0, 'review_sections': 0, 'language_retries': 0, 'fact_retries': 0, 'validation_issues': {}, 'unexpected_markers': 0, 'responses_without_markers': 0, 'actual_input_tokens': 0, 'actual_output_tokens': 0, 'source_language': source_language(markdown), 'token_samples': []} return {'sections': document, 'units': units, 'batches': batches, 'instruction': guidance, 'metrics': metrics} def summarize(markdown, prompt, invoke, *, settings, options, count=token_bound, count_method='conservative_bytes', progress=None, checkpoint=None, resume=None, context=None, reusable=None): plan = build_plan(markdown, prompt, settings, options, count, context) stats = plan['metrics'] stats['token_count_method'] = count_method saved = {} candidates = {**(reusable or {}), **((resume or {}).get('section_cache') or {})} outputs, warnings = {}, [] lock = threading.RLock() section_units = {section['id']: [] for section in plan['sections']} for unit in plan['units']: section_units[unit['section_id']].append(unit) def report(stage, percent): with lock: if progress: pieces, completed = [], 0 for section in plan['sections']: units = section_units[section['id']] # Source fallback is not a completed summary. ready = all(u['cache_key'] in saved for u in units) if section['heading']: pieces.append(section['heading']) if ready: completed += 1 pieces.extend(saved[u['cache_key']] for u in units) else: pieces.append('> 待总结') progress({'stage': stage, 'percent': percent, 'plan': dict(stats), 'partial_markdown': '\n\n'.join(pieces), 'completed_sections': completed, 'total_sections': len(plan['sections'])}) def persist(): with lock: if checkpoint: checkpoint({'version': VERSION, 'section_cache': dict(saved), 'metrics': dict(stats), 'plan': [{'ids': [u['id'] for u in batch], 'input_tokens': count(envelope(batch)), 'output_tokens': sum(u['output_estimate'] for u in batch)} for batch in plan['batches']]}) def validate(unit, value): # Ignore image labels/URLs and math in language checks, but not source text. clean = re.sub(r'!\[[^\]]*\]\([^)]*\)|AIHUBIMAGE\s*\d+\s*TOKEN|\$\$[\s\S]*?\$\$|\$[^$\n]*\$', '', value) if not language_matches(clean, unit['language']): return False, 'language' if re.search(r'(?m)^#{1,6}\s', value): return False, 'heading' valid, reason = facts_match(unit['body'], value, '\n'.join(unit['path'])) return valid, reason for unit in plan['units']: previous = candidates.get(unit['cache_key']) if isinstance(previous, str) and previous.strip() and validate(unit, previous)[0]: outputs[unit['id']] = previous saved[unit['cache_key']] = previous stats['reused_chunks'] += 1 batches = [[u for u in batch if u['id'] not in outputs] for batch in plan['batches']] batches = [batch for batch in batches if batch] stats['expected_calls'] = len(batches) + int(options.get('structure_mode') == 'reorganize' and bool(plan['units'])) stats['concurrency'] = min(stats['concurrency'], len(batches)) report('准备章节总结', 0) persist() def call(batch, repair=False): guidance = plan['instruction'] + ('\nREPAIR: previous output failed validation. Re-read only these source records; correct language, ids and numeric conditions.' if repair else '') if count(envelope(batch)) + count(guidance) + 512 > stats['input_budget']: raise ValueError('模型输入超过上下文预算') response = None for attempt in range(options.get('retries', 2) + 1): with lock: if stats['model_calls'] >= 160: raise ValueError('总结调用超过限制,已保留完成章节') stats['model_calls'] += 1 stats['expected_calls'] = max(stats['expected_calls'], stats['model_calls']) if repair: stats['repair_calls'] += 1 report('校验修复中' if repair else '模型总结中', min(89, len(outputs) * 85 // max(1, len(plan['units'])))) try: response = invoke(guidance, envelope(batch)) break except Exception as exc: if attempt == options.get('retries', 2): raise ValueError('模型调用失败,已保留完成章节,请继续处理') from exc time.sleep(attempt + 1) metadata = getattr(response, 'response_metadata', {}) or {} usage = getattr(response, 'usage_metadata', {}) or metadata.get('token_usage', {}) or {} with lock: stats['actual_input_tokens'] += int(usage.get('input_tokens', usage.get('prompt_tokens', 0)) or 0) stats['actual_output_tokens'] += int(usage.get('output_tokens', usage.get('completion_tokens', 0)) or 0) input_tokens = int(usage.get('input_tokens', usage.get('prompt_tokens', 0)) or 0) input_bytes = metadata.get('summary_input_bytes', 0) if input_tokens > 0 and input_bytes >= 4096: stats['token_samples'].append({'bytes': input_bytes, 'tokens': input_tokens}) content = getattr(response, 'content', response) if isinstance(content, list): content = ''.join(item.get('text', '') for item in content if isinstance(item, dict)) if not isinstance(content, str): content = '' content = re.sub(r'^```[^\n]*\n|\n```\s*$', '', content.strip()) truncated = (metadata.get('finish_reason') or metadata.get('done_reason')) in ('length', 'max_tokens', 'max_output_tokens') expected = {u['id'] for u in batch} # Some local models omit brackets around otherwise exact record IDs. # Normalize only IDs from this request; ordinary/unknown headings must # never be assigned to a source record by position or title. plain_ids = '|'.join(re.escape(ident) for ident in sorted(expected)) content = re.sub(r'(?m)^(#{1,6})\s+(' + plain_ids + r')\s*$', r'\1 [\2]', content) markers = list(MARKER.finditer(content)) parsed, duplicates = {}, set() with lock: stats['unexpected_markers'] += sum(m[1] not in expected for m in markers) stats['responses_without_markers'] += int(not markers) for i, marker in enumerate(markers): ident = marker[1] if ident in parsed: duplicates.add(ident) value = content[marker.end():markers[i+1].start() if i+1 < len(markers) else len(content)].strip() # An unfinished last record is never accepted. Earlier complete # records in a truncated response remain eligible for reuse. if truncated and i == len(markers) - 1: continue parsed[ident] = value failed = [] for unit in batch: value = parsed.get(unit['id'], '') valid, reason = validate(unit, value) if value and unit['id'] not in duplicates else (False, 'format') if valid: known = re.findall(r'AIHUBIMAGE\d+TOKEN', unit['body']) if known: seen = set() def image_once(match): token = match[0] if token not in known or token in seen: return '' seen.add(token) return token value = re.sub(r'AIHUBIMAGE\d+TOKEN', image_once, value) value += ''.join('\n\n' + token for token in known if token not in seen) with lock: outputs[unit['id']] = value saved[unit['cache_key']] = value else: with lock: if reason == 'language': stats['language_retries'] += 1 elif reason not in ('heading', 'format'): stats['fact_retries'] += 1 stats['validation_issues'][reason] = stats['validation_issues'].get(reason, 0) + 1 failed.append(unit) persist() report('模型总结中', min(89, len(outputs) * 85 // max(1, len(plan['units'])))) return failed def process(batch): pending = call(batch) # Retry failed records together once. Unresolved records retain their # original text; do not turn one bad batch into many model calls. for attempt in range(1): if not pending: break groups = [pending] with lock: stats['expected_calls'] += len(groups) pending = [u for group in groups for u in call(group, repair=True)] for unit in pending: # Exact source is an explicit, visible fallback, never an invented # summary or a cache hit. It preserves conditions and language. with lock: outputs[unit['id']] = unit['body'].strip() warnings.append(unit['id']) stats['fallback_sections'] += 1 persist() done = 0 if stats['concurrency'] > 1: with ThreadPoolExecutor(max_workers=stats['concurrency']) as pool: futures = [pool.submit(process, batch) for batch in batches] error = None for future in as_completed(futures): try: future.result() except Exception as exc: error = error or exc for pending in futures: pending.cancel() done += 1 report(f'总结批次 {done}/{len(batches)}', done * 90 // max(1, len(batches))) if error: raise error else: for batch in batches: process(batch) done += 1 report(f'总结批次 {done}/{len(batches)}', done * 90 // max(1, len(batches))) pieces, sources, reviews = [], [], [] for section in plan['sections']: if section['heading']: pieces.append(section['heading']) for unit in plan['units']: if unit['section_id'] != section['id']: continue value = outputs[unit['id']] missing_conditions = set(numeric_boundaries(unit['body'])) - set(numeric_boundaries(value)) review = len(unit['body']) > 512 and bool(missing_conditions) and unit['id'] not in warnings if review: reviews.append(unit['id']) pieces.append(value) sources.append({'id': unit['id'], 'section_id': section['id'], 'path': section['path'], 'start_line': unit['start'], 'end_line': unit['end'], 'source': unit['body'], 'summary': value, 'fallback': unit['id'] in warnings, 'review': review}) result = '\n\n'.join(pieces) stats['expected_calls'] = stats['model_calls'] stats['review_sections'] = len(reviews) if options.get('structure_mode') == 'reorganize': report('章节总结完成', 85) else: report('总结完成' if not warnings and not reviews else '总结完成,部分章节待核对', 100) persist() return {'text': result, 'metrics': dict(stats), 'sources': sources, 'warnings': warnings + reviews, 'state': {'section_cache': dict(saved)}}