"""Provider-neutral summary planning. No network calls or tokenizer downloads.""" import math import re from decimal import Decimal DEFAULT_SUMMARY_PROMPT = '你是文档总结助手。请将提供的正文提炼为适合思维导图展示的 Markdown。\n\n要求:\n1. 保留原文标题结构,不要重写或打乱主要标题层级。\n2. 将标题下的段落内容总结为要点,合并相近段落,避免逐段照抄。\n3. 保留原文语言:英文内容输出英文,中文内容输出中文,多语言内容按原文语言分别保留。\n4. 不要编造原文没有的信息。\n5. 保留关键数字、公式、专有名词、步骤和结论。\n6. 按本次提供的粒度、层级和节点参数整理;保留完整步骤、条件、例外和否定关系。\n7. 若当前正文是分块,结合批次范围和章节路径,仅总结本批内容,不推测其他批次内容。\n8. 节点标题尽量简短,正文说明使用短句列表。\n9. 只输出 Markdown,不要输出解释、代码块围栏或额外说明。' INTEGER_OPTIONS = { 'context_tokens': (4096, 2000000), 'max_input_tokens': (1024, 2000000), 'max_output_tokens': (256, 524288), 'output_tokens': (256, 524288), 'input_tokens': (1024, 2000000), 'safety_tokens': (256, 200000), 'timeout': (30, 600), 'retries': (0, 2), 'concurrency': (1, 8), 'max_depth': (2, 6), 'max_children': (2, 30), 'node_max_chars': (10, 300), } LOCAL_PARAMETERS = frozenset(INTEGER_OPTIONS) | {'detail', 'structure_mode', 'parallel', 'auto_split', 'tokenizer_path', 'supported_parameters', 'summary_temperature', 'summary_enable_thinking'} def validate_options(params): clean = {} for key, (low, high) in INTEGER_OPTIONS.items(): value = params.get(key) if value is None: continue if type(value) is not int or not low <= value <= high: raise ValueError(f'{key} 需要 {low} 至 {high} 的整数') clean[key] = value for key, low, high in [('temperature', 0, 2), ('top_p', 0, 1)]: value = params.get(key) if value is not None: if type(value) not in (float, int) or not math.isfinite(value) or not low <= value <= high: raise ValueError(f'{key} 无效') clean[key] = value for key, allowed in [('detail', ('brief', 'standard', 'detailed')), ('structure_mode', ('preserve', 'reorganize'))]: if params.get(key) is not None: if params[key] not in allowed: raise ValueError(f'{key} 无效') clean[key] = params[key] for key in ('parallel', 'auto_split'): if params.get(key) is not None: if type(params[key]) is not bool: raise ValueError(f'{key} 无效') clean[key] = params[key] return clean def token_counter(defaults): # Only an explicitly model-bound local tokenizer is trusted. A generic BERT # tokenizer can badly undercount a different provider's model. path = defaults.get('tokenizer_path') if path: from pathlib import Path from tokenizers import Tokenizer file = Path(path) if file.is_dir(): file /= 'tokenizer.json' tokenizer = Tokenizer.from_file(str(file)) return lambda value: len(tokenizer.encode(value, add_special_tokens=False).ids), 'model_tokenizer' return lambda value: len(value.encode('utf-8')), 'conservative_bytes' def source_language(text, prompt=''): # Summarization always preserves the source language. Old custom prompts # requesting Chinese (or saying "do not translate into Chinese") must not # disable validation. text = re.sub(r'AIHUBIMAGE\s*\d+\s*TOKEN|https?://\S+', '', text) text = re.sub(r'!\[[^\]]*\]\([^)]*\)|\$\$[\s\S]*?\$\$|\$[^$\n]*\$', '', text) for name, script in [('japanese', r'[\u3040-\u30ff]'), ('korean', r'[\uac00-\ud7af]'), ('russian', r'[\u0400-\u04ff]'), ('arabic', r'[\u0600-\u06ff]')]: if re.search(script, text): return name chinese = len(re.findall(r'[\u4e00-\u9fff]', text)) english = re.findall(r'[A-Za-z]{2,}', text) if not chinese and english: return 'english' if chinese and len(english) >= 4: return 'mixed' return 'chinese' if chinese >= 10 else 'original' def language_rule(language): return { 'english': 'OUTPUT LANGUAGE: ENGLISH ONLY. The source is English. Do not translate it into Chinese. Return Markdown headings and bullets only, without any wrapping code fence.', 'chinese': '输出语言:中文。原文中的英文术语可以保留,不要把整篇译成英文。', 'mixed': 'MULTILINGUAL SOURCE: keep EACH section in its ORIGINAL language. English sections and their bullet points MUST remain English. 中文章节和要点必须保留中文。Output BOTH languages. Never translate all sections into one language.', 'japanese': 'OUTPUT LANGUAGE: JAPANESE. Preserve the original Japanese language; do not translate.', 'korean': 'OUTPUT LANGUAGE: KOREAN. Preserve the original Korean language; do not translate.', 'russian': 'OUTPUT LANGUAGE: RUSSIAN. Preserve the original Russian language; do not translate.', 'arabic': 'OUTPUT LANGUAGE: ARABIC. Preserve the original Arabic language; do not translate.', }.get(language, '') def language_matches(text, language): scripts = {'japanese': r'[\u3040-\u30ff]', 'korean': r'[\uac00-\ud7af]', 'russian': r'[\u0400-\u04ff]', 'arabic': r'[\u0600-\u06ff]'} if language in scripts: return bool(re.search(scripts[language], text)) if language == 'english': return not re.search(r'[\u4e00-\u9fff]', text) and bool(re.search(r'[A-Za-z]{2,}', text)) if language == 'chinese': return bool(re.search(r'[\u4e00-\u9fff]', text)) if language == 'mixed': return bool(re.search(r'[\u4e00-\u9fff]', text) and re.search(r'[A-Za-z]{2,}', text)) return True def numeric_boundaries(text): """Explicit numeric comparisons only; never infer semantics via another model.""" operators = { 'do not exceed': '<=', 'does not exceed': '<=', 'no more than': '<=', 'at most': '<=', 'no less than': '>=', 'at least': '>=', 'at or above': '>=', 'at or below': '<=', 'reaches or exceeds': '>=', 'greater than': '>', 'more than': '>', 'less than': '<', 'above': '>', 'exceeds': '>', 'exceed': '>', 'below': '<', 'under': '<', 'reaches': '>=', '不超过': '<=', '不大于': '<=', '至多': '<=', '不低于': '>=', '不少于': '>=', '不小于': '>=', '至少': '>=', '达到': '>=', '超过': '>', '大于': '>', '高于': '>', '小于': '<', '低于': '<', '>=': '>=', '<=': '<=', '≥': '>=', '≤': '<=', '>': '>', '<': '<', } pattern = '|'.join(re.escape(key) for key in sorted(operators, key=len, reverse=True)) result = {} text = re.sub(r'[*_`]', '', text) text = re.sub(r'\$\$[\s\S]*?\$\$|\$[^\n$]+\$', lambda m: re.sub(r'(?<=\d)\s+(?=\d)', '', re.sub(r'(?<=\d)\s*\.\s*(?=\d)', '.', m[0])), text) for match in re.finditer(r'(?