213 lines
12 KiB
Python
213 lines
12 KiB
Python
import json
|
||
import re
|
||
from types import SimpleNamespace
|
||
from unittest import TestCase
|
||
|
||
from system_manage.summary_sections import build_plan, facts_match, sections, summarize
|
||
|
||
|
||
SETTINGS = {'context_tokens': 32768, 'output_tokens': 4096, 'safety_tokens': 1024}
|
||
OPTIONS = {'structure_mode': 'preserve', 'retries': 0}
|
||
|
||
|
||
def response(content, fn=lambda item: '- ' + item['source'].strip()):
|
||
return '\n\n'.join('### [' + item['id'] + ']\n' + fn(item) for item in json.loads(content))
|
||
|
||
|
||
class SectionSummaryTests(TestCase):
|
||
def test_exact_plain_ids_are_accepted_without_positional_matching(self):
|
||
calls = []
|
||
source = '# Operations\nInspect every 42 days.\n# Records\nKeep records for 3 years.'
|
||
def invoke(prompt, content):
|
||
calls.append(content)
|
||
return '\n\n'.join('### ' + item['id'] + '\n- ' + item['source'].strip()
|
||
for item in json.loads(content))
|
||
result = summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(len(calls), 1)
|
||
self.assertEqual(result['warnings'], [])
|
||
self.assertEqual(result['metrics']['repair_calls'], 0)
|
||
self.assertIn('42 days', result['text'])
|
||
self.assertIn('3 years', result['text'])
|
||
|
||
def test_parent_body_duplicate_titles_fences_and_source_positions(self):
|
||
source = '# Root\nParent body.\n## Same\nEnglish one.\n```md\n# example\n```\n## Same\nEnglish two.'
|
||
parsed = sections(source)
|
||
self.assertEqual(len(parsed), 3)
|
||
self.assertEqual(parsed[0]['body'], 'Parent body.\n')
|
||
self.assertNotEqual(parsed[1]['id'], parsed[2]['id'])
|
||
self.assertIn('# example', parsed[1]['body'])
|
||
self.assertEqual(parsed[2]['start'], 8)
|
||
|
||
def test_whole_document_one_batch_and_program_owns_headings(self):
|
||
calls = []
|
||
source = '# Root\nParent body.\n## Operations\nInspect every 42 days.\n## Operations\nRecord every 31 days.'
|
||
def invoke(prompt, content):
|
||
calls.append((prompt, content))
|
||
self.assertIn('NOT a translation task', prompt)
|
||
return response(content)
|
||
result = summarize(source, '请主要用中文表达', invoke, settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(len(calls), 1)
|
||
self.assertEqual(re.findall(r'(?m)^#{1,6} .+', result['text']), ['# Root', '## Operations', '## Operations'])
|
||
self.assertEqual(len(result['sources']), 3)
|
||
self.assertNotIn('### [', result['text'])
|
||
|
||
def test_only_wrong_language_section_retried(self):
|
||
calls = []
|
||
def invoke(prompt, content):
|
||
records = json.loads(content)
|
||
calls.append(records)
|
||
return response(content, lambda item: '- 中文错误' if len(calls) == 1 and 'Inspect' in item['source'] else '- ' + item['source'].strip())
|
||
result = summarize('# English\nInspect every 42 days.\n## 中文\n每31天检查一次。', '', invoke,
|
||
settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(len(calls), 2)
|
||
self.assertEqual(len(calls[1]), 1)
|
||
self.assertIn('Inspect every 42 days', result['text'])
|
||
self.assertFalse(result['warnings'])
|
||
|
||
def test_bad_ids_and_truncated_last_record_repaired_locally(self):
|
||
calls = []
|
||
def invoke(prompt, content):
|
||
records = json.loads(content)
|
||
calls.append(records)
|
||
return SimpleNamespace(content=response(content), response_metadata={'finish_reason': 'length' if len(calls) == 1 else 'stop'})
|
||
result = summarize('# A\nInspect every 42 days.\n## B\nRecord every 31 days.', '', invoke,
|
||
settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(len(calls[1]), 1)
|
||
self.assertEqual(result['metrics']['model_calls'], 2)
|
||
self.assertFalse(result['warnings'])
|
||
|
||
def test_unexpected_marker_does_not_discard_other_valid_sections(self):
|
||
calls = []
|
||
def invoke(prompt, content):
|
||
calls.append(content)
|
||
return response(content) + '\n\n### [unknown]\n- Never include this invented content 999.'
|
||
result = summarize('# A\nInspect every 42 days.\n## B\nRecord every 31 days.', '', invoke,
|
||
settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(len(calls), 1)
|
||
self.assertIn('42 days', result['text'])
|
||
self.assertIn('31 days', result['text'])
|
||
self.assertNotIn('999', result['text'])
|
||
self.assertEqual(result['metrics']['unexpected_markers'], 1)
|
||
|
||
def test_cache_survives_other_section_edit_and_line_changes(self):
|
||
states, calls = [], []
|
||
def invoke(prompt, content):
|
||
calls.append(json.loads(content))
|
||
return response(content)
|
||
source = '# Root\n## One\nInspect every 42 days.\n## Two\nRecord every 31 days.'
|
||
summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS, checkpoint=states.append, context={'model': 'v1'})
|
||
edited = source.replace('Inspect every 42 days.', 'Inspect every 43 days.\nExtra text.')
|
||
result = summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS,
|
||
reusable=states[-1]['section_cache'], context={'model': 'v1'})
|
||
self.assertEqual(len(calls[-1]), 1)
|
||
self.assertEqual(result['metrics']['reused_chunks'], 1)
|
||
calls.clear()
|
||
summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS,
|
||
reusable=states[-1]['section_cache'], context={'model': 'v2'})
|
||
self.assertEqual(len(calls[0]), 2)
|
||
|
||
def test_cache_survives_browser_line_ending_normalization(self):
|
||
states, calls = [], []
|
||
def invoke(prompt, content):
|
||
calls.append(json.loads(content))
|
||
return response(content)
|
||
source = '# Root\r\n## One\r\nInspect every 42 days.\r\n## Two\r\nRecord every 31 days.\r\n'
|
||
summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS, checkpoint=states.append)
|
||
edited = source.replace('\r\n', '\n').replace('42 days', '43 days')
|
||
result = summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS, reusable=states[-1]['section_cache'])
|
||
self.assertEqual(result['metrics']['reused_chunks'], 1)
|
||
self.assertEqual(len(calls[-1]), 1)
|
||
|
||
def test_quality_failure_retains_original_and_is_visible_not_cached(self):
|
||
states = []
|
||
result = summarize('# Operations\nInspect every 42 days.', '',
|
||
lambda p, c: response(c, lambda item: '- 每99天检查。'), settings=SETTINGS, options=OPTIONS, checkpoint=states.append)
|
||
self.assertIn('Inspect every 42 days.', result['text'])
|
||
self.assertEqual(result['metrics']['model_calls'], 2)
|
||
self.assertEqual(result['metrics']['fallback_sections'], 1)
|
||
self.assertEqual(states[-1]['section_cache'], {})
|
||
self.assertTrue(result['sources'][0]['fallback'])
|
||
|
||
def test_input_output_budget_and_no_split(self):
|
||
source = ''.join(f'## Section {i}\n' + 'Inspect every 42 days. ' * 120 + '\n' for i in range(5))
|
||
plan = build_plan(source, '', SETTINGS, {**OPTIONS, 'input_tokens': 8192})
|
||
self.assertGreater(len(plan['batches']), 1)
|
||
from system_manage.summary_sections import envelope
|
||
from system_manage.ai_hub_organize import token_bound
|
||
for batch in plan['batches']:
|
||
self.assertLessEqual(token_bound(envelope(batch)) + token_bound(plan['instruction']) + 512, 8192)
|
||
self.assertLessEqual(sum(u['output_estimate'] for u in batch), SETTINGS['output_tokens'])
|
||
with self.assertRaisesRegex(ValueError, '开启自动拆分'):
|
||
build_plan(source, '', SETTINGS, {**OPTIONS, 'input_tokens': 8192, 'auto_split': False})
|
||
|
||
def test_numeric_changes_subject_swap_units_and_large_conditions(self):
|
||
self.assertFalse(facts_match('Inspect every 42 days.', '- Inspect every 99 days.')[0])
|
||
self.assertFalse(facts_match('Inspect every 42 days.', '- Inspect every 42 hours.')[0])
|
||
self.assertFalse(facts_match('Do not restart the pump.', '- Restart the pump.')[0])
|
||
self.assertTrue(facts_match('The results are 10-50 days.', '- Results are 10–50 days.')[0])
|
||
self.assertFalse(facts_match('A above 90 degrees; B above 80 degrees.', '- A above 80 degrees; B above 90 degrees.')[0])
|
||
self.assertFalse(facts_match('; '.join('Stop above ' + str(i) for i in range(1, 10)), 'No thresholds.')[0])
|
||
self.assertTrue(facts_match('Stop above 91 degrees.', '- Stop when temperature exceeds 91.0 degrees.')[0])
|
||
|
||
def test_long_section_missing_comparison_is_visible_for_review(self):
|
||
source = '# Operations\n' + 'Inspect equipment regularly. ' * 30 + 'Stop above 91 degrees.'
|
||
result = summarize(source, '', lambda p, c: response(c, lambda item: '- Inspect equipment regularly.'), settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(result['metrics']['model_calls'], 1)
|
||
self.assertEqual(result['metrics']['review_sections'], 1)
|
||
self.assertTrue(result['sources'][0]['review'])
|
||
self.assertTrue(result['warnings'])
|
||
|
||
def test_images_stay_in_their_source_section_without_model_repair(self):
|
||
source = '# One\nEnglish text.\nAIHUBIMAGE0001TOKEN\n## Two\nOther English text.\nAIHUBIMAGE0002TOKEN'
|
||
result = summarize(source, '', lambda p, c: response(c, lambda item: '- English summary.'), settings=SETTINGS, options=OPTIONS)
|
||
self.assertEqual(result['metrics']['model_calls'], 1)
|
||
self.assertLess(result['text'].index('AIHUBIMAGE0001TOKEN'), result['text'].index('## Two'))
|
||
self.assertEqual(result['text'].count('AIHUBIMAGE0002TOKEN'), 1)
|
||
|
||
def test_error_checkpoint_reuses_successful_sibling(self):
|
||
states, calls = [], []
|
||
settings = {**SETTINGS, 'output_tokens': 1024}
|
||
source = '# One\n' + 'English fact42. ' * 150 + '\n# Two\n' + 'English fact31. ' * 150
|
||
def fail(prompt, content):
|
||
calls.append(content)
|
||
if len(calls) > 1:
|
||
raise RuntimeError('private provider error')
|
||
return response(content, lambda item: '- English fact42.')
|
||
with self.assertRaisesRegex(ValueError, '已保留完成章节'):
|
||
summarize(source, '', fail, settings=settings, options=OPTIONS, checkpoint=states.append)
|
||
result = summarize(source, '', lambda p, c: response(c, lambda item: '- English fact31.'),
|
||
settings=settings, options=OPTIONS, resume=states[-1])
|
||
self.assertGreater(result['metrics']['reused_chunks'], 0)
|
||
|
||
def test_partial_progress_and_resume_never_show_unfinished_source(self):
|
||
states, updates, calls = [], [], []
|
||
settings = {**SETTINGS, 'output_tokens': 1024}
|
||
source = '# Root\n## One\n' + 'English fact42. ' * 150 + '\n## Two\n' + 'English fact31. ' * 150
|
||
def fail(prompt, content):
|
||
calls.append(content)
|
||
if len(calls) > 1:
|
||
raise RuntimeError('failed')
|
||
return response(content, lambda item: '- English fact42.')
|
||
with self.assertRaises(ValueError):
|
||
summarize(source, '', fail, settings=settings, options=OPTIONS,
|
||
checkpoint=states.append, progress=updates.append)
|
||
self.assertEqual(updates[0]['total_sections'], 3)
|
||
self.assertIn('> 待总结', updates[0]['partial_markdown'])
|
||
self.assertNotIn('English fact31.', updates[0]['partial_markdown'])
|
||
self.assertGreater(updates[-1]['completed_sections'], 1)
|
||
resumed = []
|
||
summarize(source, '', lambda p, c: response(c, lambda item: '- English fact31.'),
|
||
settings=settings, options=OPTIONS, resume=states[-1], progress=resumed.append)
|
||
self.assertGreater(resumed[0]['completed_sections'], 1)
|
||
self.assertEqual(resumed[-1]['completed_sections'], 3)
|
||
self.assertNotIn('> 待总结', resumed[-1]['partial_markdown'])
|
||
|
||
def test_fallback_is_not_presented_as_a_completed_summary(self):
|
||
updates = []
|
||
summarize('# Operations\nInspect every 42 days.', '',
|
||
lambda p, c: response(c, lambda item: '- 每99天检查。'),
|
||
settings=SETTINGS, options=OPTIONS, progress=updates.append)
|
||
self.assertEqual(updates[-1]['completed_sections'], 0)
|
||
self.assertIn('> 待总结', updates[-1]['partial_markdown'])
|
||
self.assertNotIn('Inspect every 42 days.', updates[-1]['partial_markdown'])
|