UnisKB/apps/system_manage/test_summary_sections.py

213 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

import json
import re
from types import SimpleNamespace
from unittest import TestCase
from system_manage.summary_sections import build_plan, facts_match, sections, summarize
SETTINGS = {'context_tokens': 32768, 'output_tokens': 4096, 'safety_tokens': 1024}
OPTIONS = {'structure_mode': 'preserve', 'retries': 0}
def response(content, fn=lambda item: '- ' + item['source'].strip()):
return '\n\n'.join('### [' + item['id'] + ']\n' + fn(item) for item in json.loads(content))
class SectionSummaryTests(TestCase):
def test_exact_plain_ids_are_accepted_without_positional_matching(self):
calls = []
source = '# Operations\nInspect every 42 days.\n# Records\nKeep records for 3 years.'
def invoke(prompt, content):
calls.append(content)
return '\n\n'.join('### ' + item['id'] + '\n- ' + item['source'].strip()
for item in json.loads(content))
result = summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS)
self.assertEqual(len(calls), 1)
self.assertEqual(result['warnings'], [])
self.assertEqual(result['metrics']['repair_calls'], 0)
self.assertIn('42 days', result['text'])
self.assertIn('3 years', result['text'])
def test_parent_body_duplicate_titles_fences_and_source_positions(self):
source = '# Root\nParent body.\n## Same\nEnglish one.\n```md\n# example\n```\n## Same\nEnglish two.'
parsed = sections(source)
self.assertEqual(len(parsed), 3)
self.assertEqual(parsed[0]['body'], 'Parent body.\n')
self.assertNotEqual(parsed[1]['id'], parsed[2]['id'])
self.assertIn('# example', parsed[1]['body'])
self.assertEqual(parsed[2]['start'], 8)
def test_whole_document_one_batch_and_program_owns_headings(self):
calls = []
source = '# Root\nParent body.\n## Operations\nInspect every 42 days.\n## Operations\nRecord every 31 days.'
def invoke(prompt, content):
calls.append((prompt, content))
self.assertIn('NOT a translation task', prompt)
return response(content)
result = summarize(source, '请主要用中文表达', invoke, settings=SETTINGS, options=OPTIONS)
self.assertEqual(len(calls), 1)
self.assertEqual(re.findall(r'(?m)^#{1,6} .+', result['text']), ['# Root', '## Operations', '## Operations'])
self.assertEqual(len(result['sources']), 3)
self.assertNotIn('### [', result['text'])
def test_only_wrong_language_section_retried(self):
calls = []
def invoke(prompt, content):
records = json.loads(content)
calls.append(records)
return response(content, lambda item: '- 中文错误' if len(calls) == 1 and 'Inspect' in item['source'] else '- ' + item['source'].strip())
result = summarize('# English\nInspect every 42 days.\n## 中文\n每31天检查一次。', '', invoke,
settings=SETTINGS, options=OPTIONS)
self.assertEqual(len(calls), 2)
self.assertEqual(len(calls[1]), 1)
self.assertIn('Inspect every 42 days', result['text'])
self.assertFalse(result['warnings'])
def test_bad_ids_and_truncated_last_record_repaired_locally(self):
calls = []
def invoke(prompt, content):
records = json.loads(content)
calls.append(records)
return SimpleNamespace(content=response(content), response_metadata={'finish_reason': 'length' if len(calls) == 1 else 'stop'})
result = summarize('# A\nInspect every 42 days.\n## B\nRecord every 31 days.', '', invoke,
settings=SETTINGS, options=OPTIONS)
self.assertEqual(len(calls[1]), 1)
self.assertEqual(result['metrics']['model_calls'], 2)
self.assertFalse(result['warnings'])
def test_unexpected_marker_does_not_discard_other_valid_sections(self):
calls = []
def invoke(prompt, content):
calls.append(content)
return response(content) + '\n\n### [unknown]\n- Never include this invented content 999.'
result = summarize('# A\nInspect every 42 days.\n## B\nRecord every 31 days.', '', invoke,
settings=SETTINGS, options=OPTIONS)
self.assertEqual(len(calls), 1)
self.assertIn('42 days', result['text'])
self.assertIn('31 days', result['text'])
self.assertNotIn('999', result['text'])
self.assertEqual(result['metrics']['unexpected_markers'], 1)
def test_cache_survives_other_section_edit_and_line_changes(self):
states, calls = [], []
def invoke(prompt, content):
calls.append(json.loads(content))
return response(content)
source = '# Root\n## One\nInspect every 42 days.\n## Two\nRecord every 31 days.'
summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS, checkpoint=states.append, context={'model': 'v1'})
edited = source.replace('Inspect every 42 days.', 'Inspect every 43 days.\nExtra text.')
result = summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS,
reusable=states[-1]['section_cache'], context={'model': 'v1'})
self.assertEqual(len(calls[-1]), 1)
self.assertEqual(result['metrics']['reused_chunks'], 1)
calls.clear()
summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS,
reusable=states[-1]['section_cache'], context={'model': 'v2'})
self.assertEqual(len(calls[0]), 2)
def test_cache_survives_browser_line_ending_normalization(self):
states, calls = [], []
def invoke(prompt, content):
calls.append(json.loads(content))
return response(content)
source = '# Root\r\n## One\r\nInspect every 42 days.\r\n## Two\r\nRecord every 31 days.\r\n'
summarize(source, '', invoke, settings=SETTINGS, options=OPTIONS, checkpoint=states.append)
edited = source.replace('\r\n', '\n').replace('42 days', '43 days')
result = summarize(edited, '', invoke, settings=SETTINGS, options=OPTIONS, reusable=states[-1]['section_cache'])
self.assertEqual(result['metrics']['reused_chunks'], 1)
self.assertEqual(len(calls[-1]), 1)
def test_quality_failure_retains_original_and_is_visible_not_cached(self):
states = []
result = summarize('# Operations\nInspect every 42 days.', '',
lambda p, c: response(c, lambda item: '- 每99天检查。'), settings=SETTINGS, options=OPTIONS, checkpoint=states.append)
self.assertIn('Inspect every 42 days.', result['text'])
self.assertEqual(result['metrics']['model_calls'], 2)
self.assertEqual(result['metrics']['fallback_sections'], 1)
self.assertEqual(states[-1]['section_cache'], {})
self.assertTrue(result['sources'][0]['fallback'])
def test_input_output_budget_and_no_split(self):
source = ''.join(f'## Section {i}\n' + 'Inspect every 42 days. ' * 120 + '\n' for i in range(5))
plan = build_plan(source, '', SETTINGS, {**OPTIONS, 'input_tokens': 8192})
self.assertGreater(len(plan['batches']), 1)
from system_manage.summary_sections import envelope
from system_manage.ai_hub_organize import token_bound
for batch in plan['batches']:
self.assertLessEqual(token_bound(envelope(batch)) + token_bound(plan['instruction']) + 512, 8192)
self.assertLessEqual(sum(u['output_estimate'] for u in batch), SETTINGS['output_tokens'])
with self.assertRaisesRegex(ValueError, '开启自动拆分'):
build_plan(source, '', SETTINGS, {**OPTIONS, 'input_tokens': 8192, 'auto_split': False})
def test_numeric_changes_subject_swap_units_and_large_conditions(self):
self.assertFalse(facts_match('Inspect every 42 days.', '- Inspect every 99 days.')[0])
self.assertFalse(facts_match('Inspect every 42 days.', '- Inspect every 42 hours.')[0])
self.assertFalse(facts_match('Do not restart the pump.', '- Restart the pump.')[0])
self.assertTrue(facts_match('The results are 10-50 days.', '- Results are 10–50 days.')[0])
self.assertFalse(facts_match('A above 90 degrees; B above 80 degrees.', '- A above 80 degrees; B above 90 degrees.')[0])
self.assertFalse(facts_match('; '.join('Stop above ' + str(i) for i in range(1, 10)), 'No thresholds.')[0])
self.assertTrue(facts_match('Stop above 91 degrees.', '- Stop when temperature exceeds 91.0 degrees.')[0])
def test_long_section_missing_comparison_is_visible_for_review(self):
source = '# Operations\n' + 'Inspect equipment regularly. ' * 30 + 'Stop above 91 degrees.'
result = summarize(source, '', lambda p, c: response(c, lambda item: '- Inspect equipment regularly.'), settings=SETTINGS, options=OPTIONS)
self.assertEqual(result['metrics']['model_calls'], 1)
self.assertEqual(result['metrics']['review_sections'], 1)
self.assertTrue(result['sources'][0]['review'])
self.assertTrue(result['warnings'])
def test_images_stay_in_their_source_section_without_model_repair(self):
source = '# One\nEnglish text.\nAIHUBIMAGE0001TOKEN\n## Two\nOther English text.\nAIHUBIMAGE0002TOKEN'
result = summarize(source, '', lambda p, c: response(c, lambda item: '- English summary.'), settings=SETTINGS, options=OPTIONS)
self.assertEqual(result['metrics']['model_calls'], 1)
self.assertLess(result['text'].index('AIHUBIMAGE0001TOKEN'), result['text'].index('## Two'))
self.assertEqual(result['text'].count('AIHUBIMAGE0002TOKEN'), 1)
def test_error_checkpoint_reuses_successful_sibling(self):
states, calls = [], []
settings = {**SETTINGS, 'output_tokens': 1024}
source = '# One\n' + 'English fact42. ' * 150 + '\n# Two\n' + 'English fact31. ' * 150
def fail(prompt, content):
calls.append(content)
if len(calls) > 1:
raise RuntimeError('private provider error')
return response(content, lambda item: '- English fact42.')
with self.assertRaisesRegex(ValueError, '已保留完成章节'):
summarize(source, '', fail, settings=settings, options=OPTIONS, checkpoint=states.append)
result = summarize(source, '', lambda p, c: response(c, lambda item: '- English fact31.'),
settings=settings, options=OPTIONS, resume=states[-1])
self.assertGreater(result['metrics']['reused_chunks'], 0)
def test_partial_progress_and_resume_never_show_unfinished_source(self):
states, updates, calls = [], [], []
settings = {**SETTINGS, 'output_tokens': 1024}
source = '# Root\n## One\n' + 'English fact42. ' * 150 + '\n## Two\n' + 'English fact31. ' * 150
def fail(prompt, content):
calls.append(content)
if len(calls) > 1:
raise RuntimeError('failed')
return response(content, lambda item: '- English fact42.')
with self.assertRaises(ValueError):
summarize(source, '', fail, settings=settings, options=OPTIONS,
checkpoint=states.append, progress=updates.append)
self.assertEqual(updates[0]['total_sections'], 3)
self.assertIn('> 待总结', updates[0]['partial_markdown'])
self.assertNotIn('English fact31.', updates[0]['partial_markdown'])
self.assertGreater(updates[-1]['completed_sections'], 1)
resumed = []
summarize(source, '', lambda p, c: response(c, lambda item: '- English fact31.'),
settings=settings, options=OPTIONS, resume=states[-1], progress=resumed.append)
self.assertGreater(resumed[0]['completed_sections'], 1)
self.assertEqual(resumed[-1]['completed_sections'], 3)
self.assertNotIn('> 待总结', resumed[-1]['partial_markdown'])
def test_fallback_is_not_presented_as_a_completed_summary(self):
updates = []
summarize('# Operations\nInspect every 42 days.', '',
lambda p, c: response(c, lambda item: '- 每99天检查。'),
settings=SETTINGS, options=OPTIONS, progress=updates.append)
self.assertEqual(updates[-1]['completed_sections'], 0)
self.assertIn('> 待总结', updates[-1]['partial_markdown'])
self.assertNotIn('Inspect every 42 days.', updates[-1]['partial_markdown'])