diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader index 43ffeb85..f858c058 100644 --- a/docker/Dockerfile.docreader +++ b/docker/Dockerfile.docreader @@ -155,5 +155,8 @@ RUN mkdir -p /tmp/docreader # 暴露 gRPC 端口 EXPOSE 50051 -# 直接运行 Python 服务(日志输出到 stdout/stderr) -CMD ["uv", "run", "-m", "docreader.main"] \ No newline at end of file +# Probe ARM crypto instructions before importing native dependencies. The +# entrypoint only enables portable OpenSSL implementations after a SIGILL. +ENTRYPOINT ["/app/.venv/bin/python", "-m", "docreader.container_entrypoint"] +# Dependencies are installed from uv.lock at build time; do not mutate them at boot. +CMD ["/app/.venv/bin/python", "-m", "docreader.main"] diff --git a/docreader/container_entrypoint.py b/docreader/container_entrypoint.py new file mode 100644 index 00000000..cd0c8569 --- /dev/null +++ b/docreader/container_entrypoint.py @@ -0,0 +1,49 @@ +"""Keep ARM virtual machines with incomplete CPU support from crashing at boot. + +Probe in a child: SIGILL cannot be caught as a Python exception. Only that +specific failure permits the OpenSSL portable path; other failures stay fatal. +OPENSSL_armcap is documented at https://docs.openssl.org/master/man3/OPENSSL_armcap/ +""" + +import os +import platform +import signal +import subprocess +import sys + + +PROBE = """ +import resource +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +cipher = AESGCM(bytes(32)) +nonce = bytes(12) +encrypted = cipher.encrypt(nonce, b'docreader startup probe', b'probe') +assert cipher.decrypt(nonce, encrypted, b'probe') == b'docreader startup probe' +""" + + +def runtime_environment(): + env = os.environ.copy() + if platform.machine().lower() not in {"arm64", "aarch64"} or "OPENSSL_armcap" in env: + return env + command = [sys.executable, "-c", PROBE] + result = subprocess.run(command, env=env, capture_output=True, timeout=30) + if result.returncode == -signal.SIGILL: + env["OPENSSL_armcap"] = "0" + result = subprocess.run(command, env=env, capture_output=True, timeout=30) + if result.returncode == 0: + print("DocReader: ARM OpenSSL probe raised SIGILL; using portable CPU implementations (OPENSSL_armcap=0).", file=sys.stderr, flush=True) + if result.returncode != 0: + raise RuntimeError(f"DocReader crypto startup probe failed ({result.returncode}): {result.stderr.decode(errors='replace')}") + return env + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("DocReader entrypoint requires a command") + os.execvpe(sys.argv[1], sys.argv[1:], runtime_environment()) + + +if __name__ == "__main__": + main() diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py index 1b31db72..744a6350 100644 --- a/docreader/parser/pdf_parser.py +++ b/docreader/parser/pdf_parser.py @@ -17,6 +17,7 @@ self-sufficient using pypdfium2 + the Go-side OCR that already exists. """ import base64 +from collections import Counter import io import logging import os @@ -647,7 +648,14 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple: n = textpage.count_chars() if n <= 0: return [], 0.0 - width, height = page.get_size() + # Glyph coordinates are unrotated PDF user space. get_size() reports + # rotated dimensions and would discard valid text on cropped/rotated pages. + try: + crop_left, crop_bottom, crop_right, crop_top = page.get_cropbox() + except Exception: + width, height = page.get_size() + crop_left, crop_bottom, crop_right, crop_top = 0, 0, width, height + width = crop_right - crop_left invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else [] chars: list = [] @@ -662,7 +670,7 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple: x0, x1 = (left, right) if left <= right else (right, left) y0, y1 = (bottom, top) if bottom <= top else (top, bottom) if FILTER_HIDDEN_TEXT: - if x1 < 0 or x0 > width or y1 < 0 or y0 > height: + if x1 < crop_left or x0 > crop_right or y1 < crop_bottom or y0 > crop_top: continue # off-page glyph if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible): continue # covered by an invisible text object @@ -947,6 +955,11 @@ def _should_prefer_plain(plain: str, layout: str) -> bool: return True if not plain: return False + # Geometry must never change numeric evidence (1.5 -> 1 5, -5 -> 5). + # Compare a multiset so legitimate column reordering is still allowed. + numbers = re.compile(r"[+−-]?\d+(?:[.,:/]\d+)*(?:[%‰])?") + if Counter(numbers.findall(plain)) != Counter(numbers.findall(layout)): + return True n, single, punct_only = _layout_line_stats(layout) if n == 0: return True @@ -1489,7 +1502,7 @@ class PDFScannedParser(BaseParser): { "start": offset, "end": offset + len(line), - "locator": {"type": "pdf", "page": i + 1}, + "locator": {"type": "pdf", "page": i + 1, "mapping": "exact"}, } ) offset += len(line) @@ -1694,7 +1707,7 @@ class PDFParser(BaseParser): return start for i in range(page_count): - page_only = {"type": "pdf", "page": i + 1} + page_only = {"type": "pdf", "page": i + 1, "mapping": "exact"} if classes[i] == "scanned": page_filename = f"{base_name}_page_{i+1}.jpg" part = f"![{page_filename}](images/{page_filename})" diff --git a/docreader/parser/source_locator.py b/docreader/parser/source_locator.py index 29ef3e9b..cc3c6659 100644 --- a/docreader/parser/source_locator.py +++ b/docreader/parser/source_locator.py @@ -11,14 +11,13 @@ the origin at its top-left corner, so viewers can draw it at any zoom. from __future__ import annotations import re +from collections import Counter import statistics import unicodedata from typing import Iterable, List, Optional, Sequence, Tuple # How far past the cursor a line is searched for, in normalized characters. _SEARCH_WINDOW = 6000 -# How much of a line's normalized text is searched for. -_KEY_CHARS = 24 # Lines shorter than this (normalized) are too ambiguous to place. _MIN_KEY_CHARS = 2 # Lines shorter than this are placed only when their text is unique on the @@ -54,7 +53,22 @@ def normalize_with_positions(text: str) -> Tuple[str, List[int]]: out: List[str] = [] pos: List[int] = [] for i, ch in enumerate(text): + value = unicodedata.normalize("NFKC", ch) folded = _fold(ch) + if value in ".,:/+−-%‰<>=≤≥≠": + before, after = i - 1, i + 1 + while before >= 0 and text[before] in " \t": + before -= 1 + while after < len(text) and text[after] in " \t": + after += 1 + prev = before >= 0 and text[before].isdigit() + next_digit = after < len(text) and text[after].isdigit() + semantic = ((value in ".,/:" and prev and next_digit) + or (value in "+-−" and next_digit) + or (value in "%‰" and prev) + or value in "<>=≤≥≠") + if semantic: + folded = value.replace("−", "-") for f in folded: out.append(f) pos.append(i) @@ -62,7 +76,7 @@ def normalize_with_positions(text: str) -> Tuple[str, List[int]]: def normalize(text: str) -> str: - return "".join(_fold(ch) for ch in text) + return normalize_with_positions(text)[0] def _key_weight(key: str) -> int: @@ -81,6 +95,23 @@ def _column_cursor(boxes, idx: int, ends: List[Optional[int]]) -> Optional[int]: return None +def _numeric_safe_matches(text: str, key: str) -> List[int]: + out = [] + if not key: + return out + start = 0 + while (at := text.find(key, start)) >= 0: + before = text[at - 1] if at else "" + end = at + len(key) + after = text[end] if end < len(text) else "" + left = key[0].isdigit() and bool(before) and (before.isdigit() or before in ".,:/+−-") + right = key[-1].isdigit() and bool(after) and (after.isdigit() or after in ".,:/%‰") + if not left and not right: + out.append(at) + start = at + 1 + return out + + def locate_lines( page_text: str, line_texts: Sequence[str], @@ -100,10 +131,12 @@ def locate_lines( hits: List[Optional[int]] = [] ends: List[Optional[int]] = [] cursor = 0 + counts = Counter(normalize(line) for line in line_texts) + occurrences = {key: _numeric_safe_matches(norm, key) for key in counts} for idx, line in enumerate(line_texts): - key = normalize(line)[:_KEY_CHARS] - if len(key) < _MIN_KEY_CHARS or ( - _key_weight(key) < _SHORT_KEY_WEIGHT and norm.count(key) > 1 + key = normalize(line) + if len(key) < _MIN_KEY_CHARS or len(occurrences[key]) != counts[key] or ( + _key_weight(key) < _SHORT_KEY_WEIGHT and len(occurrences[key]) > 1 ): hits.append(None) ends.append(None) @@ -115,11 +148,11 @@ def locate_lines( cursors.insert(0, column) at = -1 for start_at in cursors: - at = norm.find(key, start_at, start_at + _SEARCH_WINDOW + len(key)) + at = next((hit for hit in occurrences[key] if start_at <= hit <= start_at + _SEARCH_WINDOW), -1) if at >= 0: break if at < 0: - at = norm.find(key) + at = occurrences[key][0] if len(occurrences[key]) == 1 else -1 if at < 0: hits.append(None) ends.append(None) @@ -232,13 +265,12 @@ def page_blocks( ``lines`` are visual lines ``{"text", "bbox"}`` (bbox in PDF points). Lines are placed in ``page_text``, grouped into paragraphs by geometry, and each - paragraph becomes a block spanning from its first line to the next - paragraph, boxed by the union of its lines. Text before the first placed - line, or a page with no placeable lines, gets a page-only block, so the - blocks always tile the page's text. + paragraph becomes a block spanning only its matched lines, boxed by their + union. Unmatched gaps keep page-only provenance and never inherit the + previous paragraph's geometry. """ end = offset + len(page_text) - page_only = {"type": "pdf", "page": page_number} + page_only = {"type": "pdf", "page": page_number, "mapping": "exact"} if not page_text: return [] placed = [] @@ -264,30 +296,49 @@ def page_blocks( for g in (_line_gap(a[1], b[1]) for a, b in zip(placed, placed[1:])) if g is not None and g >= 0 ] - usual_gap = statistics.median(gaps) if gaps else line_h * 0.5 + usual_gap = min(statistics.median(gaps), line_h * 1.5) if gaps else line_h * 0.5 + + norm, positions = normalize_with_positions(page_text) + def line_end(at, line): + key = normalize(line["text"]) + norm_start = next((i for i, p in enumerate(positions) if p >= at), len(norm)) + hit = norm.find(key, norm_start) + if hit < 0: + return at + end_at = positions[hit + len(key) - 1] + 1 + while end_at < len(page_text) and not page_text[end_at].isalnum(): + if page_text[end_at] == "\n": + return end_at + 1 + end_at += 1 + return end_at paragraphs: List[Tuple[int, List[dict]]] = [] for at, ln in placed: + ln = {**ln, "_source_end": line_end(at, ln)} if paragraphs: prev = paragraphs[-1][1][-1] col_right = max(p["bbox"][2] for p in paragraphs[-1][1]) col_right = max(col_right, ln["bbox"][2]) - if not _starts_paragraph(prev, ln, line_h, usual_gap, col_right): + previous_end = prev["_source_end"] + gap_is_empty = not normalize(page_text[previous_end:at]) + if gap_is_empty and not _starts_paragraph(prev, ln, line_h, usual_gap, col_right): paragraphs[-1][1].append(ln) continue paragraphs.append((at, [ln])) blocks: List[dict] = [] - first_at = paragraphs[0][0] - if first_at > 0 and page_text[:first_at].strip(): - blocks.append({"start": offset, "end": offset + first_at, "locator": dict(page_only)}) - for i, (at, members) in enumerate(paragraphs): - stop = paragraphs[i + 1][0] if i + 1 < len(paragraphs) else len(page_text) - if i == 0: - at = 0 if not blocks else at + cursor = 0 + for at, members in paragraphs: + stop = members[-1]["_source_end"] if stop <= at: continue + if at > cursor and page_text[cursor:at].strip(): + blocks.append({"start": offset + cursor, "end": offset + at, "locator": dict(page_only)}) locator = dict(page_only) locator["bbox"] = geometry.bbox(_union(m["bbox"] for m in members)) + locator["source_id"] = f"pdf:{page_number}:{at}:{stop}" blocks.append({"start": offset + at, "end": offset + stop, "locator": locator}) + cursor = stop + if cursor < len(page_text) and page_text[cursor:].strip(): + blocks.append({"start": offset + cursor, "end": end, "locator": dict(page_only)}) return blocks diff --git a/docreader/tests/generate_source_location_browser_fixtures.py b/docreader/tests/generate_source_location_browser_fixtures.py new file mode 100644 index 00000000..a5365034 --- /dev/null +++ b/docreader/tests/generate_source_location_browser_fixtures.py @@ -0,0 +1,41 @@ +"""Regenerate browser fixtures with the actual builtin PDF parser. + +Run from the repository root: + docreader/.venv/bin/python -m docreader.tests.generate_source_location_browser_fixtures +""" +import base64 +import json +from pathlib import Path +from unittest.mock import patch + +from docreader.parser.pdf_parser import PDFParser +from docreader.tests.test_source_locator_pdf import pdf_fixture + + +def main(): + cases = [{"name": "PDFium lower paragraph with decimal evidence", "page": 2, + "needle": "Second page lower evidence", "pages": [ + {"lines": [("First page has independent original evidence.", 60, 700)]}, + {"lines": [("Second page upper paragraph is unrelated.", 60, 700), + ("Second page lower evidence: the limit is 1.5 mm.", 60, 400)]}]}] + for rotation in (0, 90, 180, 270): + cases.append({"name": f"PDFium CropBox and rotation {rotation}", "page": 1, + "needle": "Cropped and rotated", "pages": [ + {"rotation": rotation, "crop": "/CropBox [100 100 500 700]", + "lines": [("Cropped and rotated source evidence.", 150, 600)]}]}) + output = [] + for case in cases: + data = pdf_fixture(case["pages"]) + with patch("docreader.parser.pdf_parser.SOURCE_BOXES", True): + doc = PDFParser(file_name="fixture.pdf", pdf_force_scanned=False).parse_into_text(data) + at = doc.content.index(case["needle"]) + block = next(b for b in doc.source_blocks if b["start"] <= at < b["end"]) + locator = {**block["locator"], "quote": doc.content[block["start"]:block["end"]].strip()} + assert locator.get("bbox"), case["name"] + output.append({"name": case["name"], "page": case["page"], "pdf": base64.b64encode(data).decode(), "locator": locator}) + target = Path(__file__).resolve().parents[2] / "frontend/tests/source-locate/parsedFixtures.json" + target.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docreader/tests/test_container_entrypoint.py b/docreader/tests/test_container_entrypoint.py new file mode 100644 index 00000000..49a42dac --- /dev/null +++ b/docreader/tests/test_container_entrypoint.py @@ -0,0 +1,64 @@ +import os +import signal +import subprocess +import unittest +from unittest.mock import patch + +from docreader.container_entrypoint import runtime_environment + + +class ContainerEntrypointTest(unittest.TestCase): + def setUp(self): + env = patch.dict(os.environ, {"PATH": "/app/.venv/bin"}, clear=True) + env.start() + self.addCleanup(env.stop) + machine = patch("docreader.container_entrypoint.platform.machine", return_value="aarch64") + machine.start() + self.addCleanup(machine.stop) + + @patch("docreader.container_entrypoint.subprocess.run") + def test_healthy_arm_keeps_acceleration(self, run): + run.return_value = subprocess.CompletedProcess([], 0, b"", b"") + self.assertNotIn("OPENSSL_armcap", runtime_environment()) + self.assertEqual(run.call_count, 1) + + @patch("docreader.container_entrypoint.subprocess.run") + def test_sigill_retries_and_verifies_portable_crypto(self, run): + run.side_effect = [subprocess.CompletedProcess([], -signal.SIGILL, b"", b""), subprocess.CompletedProcess([], 0, b"", b"")] + self.assertEqual(runtime_environment()["OPENSSL_armcap"], "0") + self.assertEqual(run.call_count, 2) + self.assertNotIn("OPENSSL_armcap", os.environ) + + @patch("docreader.container_entrypoint.subprocess.run") + def test_other_failure_is_not_masked(self, run): + run.return_value = subprocess.CompletedProcess([], 1, b"", b"missing dependency") + with self.assertRaisesRegex(RuntimeError, "missing dependency"): + runtime_environment() + self.assertEqual(run.call_count, 1) + + @patch("docreader.container_entrypoint.subprocess.run") + def test_failed_portable_probe_stays_fatal(self, run): + run.side_effect = [subprocess.CompletedProcess([], -signal.SIGILL, b"", b""), subprocess.CompletedProcess([], 1, b"", b"encryption failed")] + with self.assertRaisesRegex(RuntimeError, "encryption failed"): + runtime_environment() + + @patch("docreader.container_entrypoint.subprocess.run") + def test_operator_override_is_preserved(self, run): + os.environ["OPENSSL_armcap"] = "0x1" + self.assertEqual(runtime_environment()["OPENSSL_armcap"], "0x1") + run.assert_not_called() + + @patch("docreader.container_entrypoint.platform.machine", return_value="x86_64") + @patch("docreader.container_entrypoint.subprocess.run") + def test_other_architectures_are_unchanged(self, run, machine): + self.assertNotIn("OPENSSL_armcap", runtime_environment()) + run.assert_not_called() + + @patch("docreader.container_entrypoint.subprocess.run", side_effect=subprocess.TimeoutExpired("probe", 30)) + def test_probe_has_bounded_startup(self, run): + with self.assertRaises(subprocess.TimeoutExpired): + runtime_environment() + + +if __name__ == "__main__": + unittest.main() diff --git a/docreader/tests/test_source_locator.py b/docreader/tests/test_source_locator.py index f84cf2f4..bbdbf18b 100644 --- a/docreader/tests/test_source_locator.py +++ b/docreader/tests/test_source_locator.py @@ -63,7 +63,7 @@ class PageBlocksTest(unittest.TestCase): def test_page_without_lines_gets_page_block(self): blocks = page_blocks("文字", 5, 2, [], None) - self.assertEqual(blocks, [{"start": 5, "end": 7, "locator": {"type": "pdf", "page": 2}}]) + self.assertEqual(blocks, [{"start": 5, "end": 7, "locator": {"type": "pdf", "page": 2, "mapping": "exact"}}]) class WireTest(unittest.TestCase): @@ -139,5 +139,41 @@ class SplitLineAtGapsTest(unittest.TestCase): self.assertEqual(len(_split_line_at_gaps(self._line(chars))), 1) + +class PrecisionRegressionTest(unittest.TestCase): + def test_common_prefix_cannot_place_a_different_conclusion(self): + prefix = "Common opening repeated before the important conclusion " + self.assertEqual(locate_lines(prefix + "failed", [prefix + "passed"]), [None]) + + def test_numeric_meaning_is_preserved(self): + for source, quote in [("limit 15 mm", "limit 1.5 mm"), ("temperature 5 C", "temperature -5 C"), ("rate 5", "rate 5%")]: + self.assertEqual(locate_lines(source, [quote]), [None]) + + def test_unaccounted_duplicate_does_not_take_first_region(self): + quote = "A repeated paragraph with enough text" + self.assertEqual(locate_lines(quote + "\n" + quote, [quote]), [None]) + + def test_unmatched_text_never_inherits_previous_geometry(self): + page = "First matched line.\nUnmatched evidence in between.\nLast matched line." + lines = [_ln("First matched line.", 50, 700, 300, 712), _ln("Last matched line.", 50, 668, 300, 680)] + blocks = page_blocks(page, 0, 2, lines, PageGeometry((0, 0, 600, 800))) + at = page.index("Unmatched") + covering = [b for b in blocks if b["start"] <= at < b["end"]] + self.assertEqual(len(covering), 1) + self.assertNotIn("bbox", covering[0]["locator"]) + self.assertEqual(covering[0]["locator"]["page"], 2) + + def test_trailing_unmatched_text_has_only_page_provenance(self): + page = "Matched paragraph.\nUnrecognized trailing evidence." + blocks = page_blocks(page, 0, 7, [_ln("Matched paragraph.", 50, 700, 300, 712)], PageGeometry((0, 0, 600, 800))) + self.assertEqual(blocks[0]["end"], page.index("Unrecognized")) + self.assertNotIn("bbox", blocks[-1]["locator"]) + + def test_wire_preserves_verified_geometry_and_identity(self): + from docreader.source_wire import source_blocks_to_proto + locator = {"type": "pdf", "page": 501, "mapping": "exact", "source_id": "pdf:501:0:3", "bbox": [.1, .2, .8, .4]} + wire = source_blocks_to_proto(Document(content="abc", source_blocks=[{"start": 0, "end": 3, "locator": locator}])) + self.assertEqual(json.loads(wire[0].locator_json), locator) + if __name__ == "__main__": unittest.main() diff --git a/docreader/tests/test_source_locator_pdf.py b/docreader/tests/test_source_locator_pdf.py new file mode 100644 index 00000000..daa652c9 --- /dev/null +++ b/docreader/tests/test_source_locator_pdf.py @@ -0,0 +1,78 @@ +"""Actual PDFium parsing through source blocks and the protobuf JSON boundary.""" +import json +import unittest +from unittest.mock import patch + +from docreader.parser.pdf_parser import PDFParser +from docreader.source_wire import source_blocks_to_proto +from docreader.tests.test_pdf_embedded_images import _pdf_from_objects + + +def pdf_fixture(pages): + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + ("<< /Type /Pages /Count %d /Kids [%s] >>" % ( + len(pages), " ".join(f"{4+i*2} 0 R" for i in range(len(pages))) + )).encode(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + for i, page in enumerate(pages): + stream = "\n".join(f"BT /F1 14 Tf {x} {y} Td ({text}) Tj ET" for text, x, y in page["lines"]).encode() + objects.extend([ + (f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate {page.get('rotation', 0)} " + f"{page.get('crop', '')} /Resources << /Font << /F1 3 0 R >> >> /Contents {5+i*2} 0 R >>").encode(), + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + ]) + return _pdf_from_objects(objects) + + +class NativePDFSourceTest(unittest.TestCase): + def parse(self, pages): + with patch("docreader.parser.pdf_parser.SOURCE_BOXES", True): + doc = PDFParser(file_name="source-regression.pdf", file_type="pdf", pdf_force_scanned=False).parse_into_text(pdf_fixture(pages)) + self.assertFalse(doc.metadata.get("scanned_page_count")) + return doc + + def test_page_and_region_survive_actual_parser_and_wire(self): + doc = self.parse([ + {"lines": [("First page has independent original evidence.", 60, 700)]}, + {"lines": [("Second page upper paragraph is unrelated.", 60, 700), + ("Second page lower evidence: the limit is 1.5 mm.", 60, 400)]}, + ]) + at = doc.content.index("Second page lower evidence") + source = [b for b in doc.source_blocks if b["start"] <= at < b["end"]] + self.assertEqual(len(source), 1) + loc = source[0]["locator"] + self.assertEqual(loc["page"], 2) + self.assertEqual(loc["mapping"], "exact") + self.assertGreater(loc["bbox"][1], .45) + self.assertLess(loc["bbox"][3], .55) + self.assertIn("1.5 mm", doc.content[source[0]["start"]:source[0]["end"]]) + self.assertIn(loc, [json.loads(b.locator_json) for b in source_blocks_to_proto(doc)]) + + def test_cropbox_and_rotation_use_displayed_page_coordinates(self): + for rotation in (0, 90, 180, 270): + with self.subTest(rotation=rotation): + doc = self.parse([{"rotation": rotation, "crop": "/CropBox [100 100 500 700]", + "lines": [("Cropped and rotated source evidence.", 150, 600)]}]) + loc = next(b["locator"] for b in doc.source_blocks if b["locator"].get("bbox")) + x0, y0, x1, y1 = loc["bbox"] + self.assertTrue(0 <= x0 < x1 <= 1 and 0 <= y0 < y1 <= 1) + if rotation == 0: self.assertLess(y1, .2) + elif rotation == 90: self.assertGreater(x0, .8) + elif rotation == 180: self.assertGreater(y0, .8) + else: self.assertLess(x1, .2) + + def test_identical_paragraphs_keep_distinct_vertical_regions(self): + text = "Repeated paragraph with identical original evidence." + doc = self.parse([{"lines": [(text, 60, 700), (text, 60, 400)]}]) + boxed = [b for b in doc.source_blocks if b["locator"].get("bbox")] + self.assertEqual(len(boxed), 2) + self.assertNotEqual(boxed[0]["locator"]["source_id"], boxed[1]["locator"]["source_id"]) + self.assertLess(boxed[0]["locator"]["bbox"][1], .2) + self.assertGreater(boxed[1]["locator"]["bbox"][1], .45) + self.assertIn(text.rstrip('.'), doc.content[boxed[1]["start"]:boxed[1]["end"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/components/ChatReferenceSourceView.vue b/frontend/src/components/ChatReferenceSourceView.vue index 38c2931b..4e3a5bbc 100644 --- a/frontend/src/components/ChatReferenceSourceView.vue +++ b/frontend/src/components/ChatReferenceSourceView.vue @@ -58,6 +58,8 @@ + + + + diff --git a/frontend/tests/source-locate/README.md b/frontend/tests/source-locate/README.md new file mode 100644 index 00000000..ac02ee40 --- /dev/null +++ b/frontend/tests/source-locate/README.md @@ -0,0 +1,60 @@ +# Source location rendering regressions + +From `frontend`, run `npm run dev -- --host 127.0.0.1 --port 5199`, then open +`http://127.0.0.1:5199/tests/source-locate/index.html` in a browser. +The page runs 52 cases against the actual Vue previews, PDF.js, docx-preview, +SheetJS and EPUB renderer. Each result is displayed as PASS/FAIL. The page is +outside the production entry point and is not included in the app build. + +The five `parsedFixtures.json` cases are generated by the real Python PDFium +parser. Regenerate from the repository root after changing parsing or geometry: + +```sh +docreader/.venv/bin/python -m docreader.tests.generate_source_location_browser_fixtures +``` + +The fixtures contain only generated test text. No user documents or remote +parser services are needed. Assertions check page identity, ambiguity rejection, +nonempty finite geometry, region containment, all fragments of cross-page +citations, stale evidence, and rapid document destruction/recreation. + +The pure resolver's separate Node tests cover page 501, cancellation, partial +mapping, numeric meaning, and DOMRect prototype properties. Python tests cover +native PDF parsing, crop/rotation, duplicate paragraphs and protobuf transport. + +## Private local replay + +Open the same page with `?local=1` to replay `local/manifest.json`. The `local/` +directory ignores all fixture files, so uploaded documents and query logs are +never added to Git. Each manifest entry provides `name`, `file`, `ext`, +`request` (a `SourceLocateRequest`), expected `precise`, and expected PDF +`pages`, and optional `firstPage` for the initial navigation target. Files are served only by the local development server. The replay +checks actual highlighted pages and precision status, not merely whether a +preview opened. Keep real user files out of `parsedFixtures.json`. + +The generated DOCX cases also check an OCR image bracketed by unique source +paragraphs, missing/duplicate image contexts, images at chunk boundaries, +and a valid sentence within a partially mapped chunk. + +PDF cases include column-serialized tables with repeated course names and +paired row values, and image-heavy cross-page instructions with corrupted font +arrows. Separate resolver tests reject wrong grades, mismatched rows, duplicate +tables, and numeric changes. Partial highlights never claim full chunk coverage. + +DOCX regressions also verify exact image-byte identity, ambiguous duplicate images, +and private replay assertions for the exact highlighted excerpt and image digest. + +Markdown regressions render cited raw Markdown using the same sanitized renderer +as the preview. They cover fenced language names, literal tags inside code, +exact commands, paragraph-level paraphrases, multi-paragraph citations, ambiguous +duplicate sources, missing source scopes, and changed numeric values. DOM anchor +cases verify that a citation keeps all preceding sentences since the previous +citation. Private Markdown fixtures must provide `request.sourceMarkdown` before +markup stripping; `highlightText` asserts the exact highlighted source paragraphs. + +Chunk-boundary cases assert complete original paragraph context when a child +ends or starts mid-paragraph. Candidate paragraphs must intersect a uniquely +verified source region with at least eight normalized characters. Duplicate +source scopes, modified values and unrelated adjacent paragraphs cannot use this +completion path. A repeated paragraph keeps the occurrence established by the +full source scope. Exact explicit quotations still retain their narrow range. diff --git a/frontend/tests/source-locate/index.html b/frontend/tests/source-locate/index.html new file mode 100644 index 00000000..4794f478 --- /dev/null +++ b/frontend/tests/source-locate/index.html @@ -0,0 +1,3 @@ + +Source location regression tests +
diff --git a/frontend/tests/source-locate/local/.gitignore b/frontend/tests/source-locate/local/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/frontend/tests/source-locate/local/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/frontend/tests/source-locate/main.ts b/frontend/tests/source-locate/main.ts new file mode 100644 index 00000000..1b16c5eb --- /dev/null +++ b/frontend/tests/source-locate/main.ts @@ -0,0 +1,9 @@ +import { createApp } from 'vue' +import { createI18n } from 'vue-i18n' +import { Button, Loading } from 'tdesign-vue-next' +import { Icon } from 'tdesign-icons-vue-next' +import Harness from './Harness.vue' +createApp(Harness).component('t-button', Button).component('t-loading', Loading).component('t-icon', Icon).use(createI18n({ legacy: false, locale: 'en', messages: { en: { + preview: { zoomOut: 'Zoom out', zoomIn: 'Zoom in', loadFailed: 'Load failed' }, + chat: { referenceSourcePrevious: 'Previous location', referenceSourceNext: 'Next location' }, +} } })).mount('#app') diff --git a/frontend/tests/source-locate/officeFixtures.ts b/frontend/tests/source-locate/officeFixtures.ts new file mode 100644 index 00000000..ca48d483 --- /dev/null +++ b/frontend/tests/source-locate/officeFixtures.ts @@ -0,0 +1,36 @@ +import JSZip from 'jszip' +import * as XLSX from 'xlsx' + +export async function docxFixture(): Promise { + const zip = new JSZip() + zip.file('[Content_Types].xml', '') + zip.file('_rels/.rels', '') + zip.file('word/document.xml', 'Unrelated first paragraph.The cited limit is 1.5 millimeters.') + return zip.generateAsync({ type: 'blob', mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }) +} + +export function sheetFixture(): Blob { + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, XLSX.utils.aoa_to_sheet([['Equipment', 'Limit'], ['Anchor', '1.5 mm'], ['Cable', '15 mm']]), 'Inspection') + return new Blob([XLSX.write(book, { type: 'array', bookType: 'xlsx' })]) +} + +export async function epubFixture(): Promise { + const zip = new JSZip() + zip.file('mimetype', 'application/epub+zip') + zip.file('META-INF/container.xml', '') + zip.file('OEBPS/content.opf', '') + zip.file('OEBPS/one.xhtml', '

Repeated cited passage.

') + zip.file('OEBPS/two.xhtml', '

Repeated cited passage.

') + return zip.generateAsync({ type: 'blob', mimeType: 'application/epub+zip' }) +} + +export async function docxImageFixture(count = 1): Promise { + const zip = await JSZip.loadAsync(await (await docxFixture()).arrayBuffer()) + zip.file('word/_rels/document.xml.rels', '') + zip.file('word/media/image.png', 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', { base64: true }) + const drawing = '' + const xml = await zip.file('word/document.xml')!.async('string') + zip.file('word/document.xml', xml.replace('', drawing.repeat(count) + 'Verified paragraph after the image.')) + return zip.generateAsync({ type: 'blob' }) +} diff --git a/frontend/tests/source-locate/parsedFixtures.json b/frontend/tests/source-locate/parsedFixtures.json new file mode 100644 index 00000000..7f65b07a --- /dev/null +++ b/frontend/tests/source-locate/parsedFixtures.json @@ -0,0 +1,92 @@ +[ + { + "name": "PDFium lower paragraph with decimal evidence", + "page": 2, + "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAyIC9LaWRzIFs0IDAgUiA2IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAwICAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSAzIDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjUgMCBvYmoKPDwgL0xlbmd0aCA3NiA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDYwIDcwMCBUZCAoRmlyc3QgcGFnZSBoYXMgaW5kZXBlbmRlbnQgb3JpZ2luYWwgZXZpZGVuY2UuKSBUaiBFVAplbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MDAgODAwXSAvUm90YXRlIDAgIC9SZXNvdXJjZXMgPDwgL0ZvbnQgPDwgL0YxIDMgMCBSID4+ID4+IC9Db250ZW50cyA3IDAgUiA+PgplbmRvYmoKNyAwIG9iago8PCAvTGVuZ3RoIDE1MiA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDYwIDcwMCBUZCAoU2Vjb25kIHBhZ2UgdXBwZXIgcGFyYWdyYXBoIGlzIHVucmVsYXRlZC4pIFRqIEVUCkJUIC9GMSAxNCBUZiA2MCA0MDAgVGQgKFNlY29uZCBwYWdlIGxvd2VyIGV2aWRlbmNlOiB0aGUgbGltaXQgaXMgMS41IG1tLikgVGogRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgOAowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTIxIDAwMDAwIG4gCjAwMDAwMDAxOTEgMDAwMDAgbiAKMDAwMDAwMDMyOCAwMDAwMCBuIAowMDAwMDAwNDU0IDAwMDAwIG4gCjAwMDAwMDA1OTEgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA4IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo3OTQKJSVFT0YK", + "locator": { + "type": "pdf", + "page": 2, + "mapping": "exact", + "bbox": [ + 0.1, + 0.4842, + 0.6071, + 0.5037 + ], + "source_id": "pdf:2:42:90", + "quote": "Second page lower evidence: the limit is 1.5 mm." + } + }, + { + "name": "PDFium CropBox and rotation 0", + "page": 1, + "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAwIC9Dcm9wQm94IFsxMDAgMTAwIDUwMCA3MDBdIC9SZXNvdXJjZXMgPDwgL0ZvbnQgPDwgL0YxIDMgMCBSID4+ID4+IC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNSAwIG9iago8PCAvTGVuZ3RoIDY4ID4+CnN0cmVhbQpCVCAvRjEgMTQgVGYgMTUwIDYwMCBUZCAoQ3JvcHBlZCBhbmQgcm90YXRlZCBzb3VyY2UgZXZpZGVuY2UuKSBUaiBFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDE4NSAwMDAwMCBuIAowMDAwMDAwMzQ4IDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKNDY2CiUlRU9GCg==", + "locator": { + "type": "pdf", + "page": 1, + "mapping": "exact", + "bbox": [ + 0.125, + 0.1456, + 0.7203, + 0.1716 + ], + "source_id": "pdf:1:0:36", + "quote": "Cropped and rotated source evidence." + } + }, + { + "name": "PDFium CropBox and rotation 90", + "page": 1, + "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSA5MCAvQ3JvcEJveCBbMTAwIDEwMCA1MDAgNzAwXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSAzIDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjUgMCBvYmoKPDwgL0xlbmd0aCA2OCA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDE1MCA2MDAgVGQgKENyb3BwZWQgYW5kIHJvdGF0ZWQgc291cmNlIGV2aWRlbmNlLikgVGogRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAxODUgMDAwMDAgbiAKMDAwMDAwMDM0OSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQ2NwolJUVPRgo=", + "locator": { + "type": "pdf", + "page": 1, + "mapping": "exact", + "bbox": [ + 0.8284, + 0.125, + 0.8544, + 0.7203 + ], + "source_id": "pdf:1:0:36", + "quote": "Cropped and rotated source evidence." + } + }, + { + "name": "PDFium CropBox and rotation 180", + "page": 1, + "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAxODAgL0Nyb3BCb3ggWzEwMCAxMDAgNTAwIDcwMF0gL1Jlc291cmNlcyA8PCAvRm9udCA8PCAvRjEgMyAwIFIgPj4gPj4gL0NvbnRlbnRzIDUgMCBSID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNjggPj4Kc3RyZWFtCkJUIC9GMSAxNCBUZiAxNTAgNjAwIFRkIChDcm9wcGVkIGFuZCByb3RhdGVkIHNvdXJjZSBldmlkZW5jZS4pIFRqIEVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMTg1IDAwMDAwIG4gCjAwMDAwMDAzNTAgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0NjgKJSVFT0YK", + "locator": { + "type": "pdf", + "page": 1, + "mapping": "exact", + "bbox": [ + 0.2797, + 0.8284, + 0.875, + 0.8544 + ], + "source_id": "pdf:1:0:36", + "quote": "Cropped and rotated source evidence." + } + }, + { + "name": "PDFium CropBox and rotation 270", + "page": 1, + "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAyNzAgL0Nyb3BCb3ggWzEwMCAxMDAgNTAwIDcwMF0gL1Jlc291cmNlcyA8PCAvRm9udCA8PCAvRjEgMyAwIFIgPj4gPj4gL0NvbnRlbnRzIDUgMCBSID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNjggPj4Kc3RyZWFtCkJUIC9GMSAxNCBUZiAxNTAgNjAwIFRkIChDcm9wcGVkIGFuZCByb3RhdGVkIHNvdXJjZSBldmlkZW5jZS4pIFRqIEVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMTg1IDAwMDAwIG4gCjAwMDAwMDAzNTAgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0NjgKJSVFT0YK", + "locator": { + "type": "pdf", + "page": 1, + "mapping": "exact", + "bbox": [ + 0.1456, + 0.2797, + 0.1716, + 0.875 + ], + "source_id": "pdf:1:0:36", + "quote": "Cropped and rotated source evidence." + } + } +] diff --git a/frontend/tests/source-locate/pdfFixture.ts b/frontend/tests/source-locate/pdfFixture.ts new file mode 100644 index 00000000..fb297a63 --- /dev/null +++ b/frontend/tests/source-locate/pdfFixture.ts @@ -0,0 +1,20 @@ +/** Tiny real PDF fixture, generated without a PDF library or network service. */ +export function fixturePdf(pages: Array<{ lines: Array<[string, number, number]>; rotation?: number }>): ArrayBuffer { + const objects: string[] = [] + objects[0] = '<< /Type /Catalog /Pages 2 0 R >>' + objects[1] = `<< /Type /Pages /Count ${pages.length} /Kids [${pages.map((_, i) => `${4 + i * 2} 0 R`).join(' ')}] >>` + objects[2] = '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>' + pages.forEach((page, i) => { + const stream = page.lines.map(([text, x, y]) => `BT /F1 14 Tf ${x} ${y} Td (${text.replace(/[\\()]/g, '\\$&')}) Tj ET`).join('\n') + objects[3 + i * 2] = `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate ${page.rotation || 0} /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + i * 2} 0 R >>` + objects[4 + i * 2] = `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream` + }) + let pdf = '%PDF-1.4\n' + const offsets = [0] + objects.forEach((object, i) => { offsets.push(pdf.length); pdf += `${i + 1} 0 obj\n${object}\nendobj\n` }) + const xref = pdf.length + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n` + pdf += offsets.slice(1).map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`).join('') + pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF` + return new TextEncoder().encode(pdf).buffer +} diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index cf5ada5e..6b6d340c 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -414,6 +414,7 @@ func (r *chunkRepository) SaveChunkRevision( Updates(map[string]interface{}{ "content": common.CleanInvalidUTF8(chunk.Content), "source_content": common.CleanInvalidUTF8(chunk.SourceContent), + "source_locators": chunk.SourceLocators, "content_revision": chunk.ContentRevision, "is_enabled": chunk.IsEnabled, "metadata": chunk.Metadata, diff --git a/internal/application/repository/chunk_revision_test.go b/internal/application/repository/chunk_revision_test.go index c8c518ac..4226ed3b 100644 --- a/internal/application/repository/chunk_revision_test.go +++ b/internal/application/repository/chunk_revision_test.go @@ -23,9 +23,20 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) { chunk := &types.Chunk{ ID: uuid.NewString(), TenantID: 1, KnowledgeBaseID: "kb", KnowledgeID: "knowledge", Content: "before", SourceContent: "before", ChunkType: types.ChunkTypeText, + SourceLocators: types.SourceLocators{{ + Type: "pdf", + Page: 1, + Mapping: "exact", + SourceID: "block:1", + SourceHash: "revision-one", + Partial: true, + }}, IsEnabled: true, IndexStatus: "ready", CreatedAt: now, UpdatedAt: now, } require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{chunk})) + initial, err := repo.GetChunkByID(ctx, 1, chunk.ID) + require.NoError(t, err) + require.Equal(t, chunk.SourceLocators, initial.SourceLocators) snapshot := &types.ChunkRevision{ ID: uuid.NewString(), TenantID: 1, KnowledgeBaseID: "kb", KnowledgeID: "knowledge", @@ -33,12 +44,14 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) { EditSource: "user", EditedAt: now, CreatedAt: now, } chunk.Content = "after" + chunk.SourceLocators = nil chunk.ContentRevision = 1 require.NoError(t, repo.SaveChunkRevision(ctx, chunk, snapshot, 0)) stored, err := repo.GetChunkByID(ctx, 1, chunk.ID) require.NoError(t, err) require.Equal(t, "after", stored.Content) + require.Empty(t, stored.SourceLocators) require.Equal(t, 1, stored.ContentRevision) revisions, err := repo.ListChunkRevisions(ctx, 1, chunk.ID) require.NoError(t, err) @@ -55,6 +68,7 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) { stored, err = repo.GetChunkByID(ctx, 1, chunk.ID) require.NoError(t, err) require.Equal(t, "after", stored.Content) + require.Empty(t, stored.SourceLocators) count := int64(0) require.NoError(t, db.Model(&types.ChunkRevision{}).Count(&count).Error) require.Equal(t, int64(1), count) diff --git a/internal/application/service/chat_pipeline/citation_sources.go b/internal/application/service/chat_pipeline/citation_sources.go new file mode 100644 index 00000000..266dfc14 --- /dev/null +++ b/internal/application/service/chat_pipeline/citation_sources.go @@ -0,0 +1,192 @@ +package chatpipeline + +import ( + "context" + "sort" + "strings" + + "github.com/Tencent/WeKnora/internal/searchutil" + "github.com/Tencent/WeKnora/internal/types" +) + +// Ranking works on expanded contexts. The model must instead see every source +// body under its own ID; a parent's neighbouring paragraph is not child evidence. +func (p *PluginMerge) attachCitationSources(ctx context.Context, results []*types.SearchResult) { + if p.chunkRepo == nil { + return + } + ids := map[string]bool{} + for _, r := range results { + if len(r.SubChunkID) > 0 { + r.ContentRewritten = true + } + if r.ContentRewritten { + ids[r.ID] = true + for _, id := range r.SubChunkID { + ids[id] = true + } + } + } + if len(ids) == 0 { + return + } + keys := make([]string, 0, len(ids)) + for id := range ids { + keys = append(keys, id) + } + chunks, err := p.chunkRepo.ListChunksByIDOnly(ctx, keys) + if err != nil { + return + } + byID := map[string]*types.Chunk{} + parents := map[string]bool{} + for _, c := range chunks { + byID[c.ID] = c + if c.ParentChunkID != "" { + parents[c.ParentChunkID] = true + } + } + keys = nil + for id := range parents { + keys = append(keys, id) + } + // Image -> text -> parent_text. Fetch the intermediate text parent before + // collecting siblings. All IDs originate in authorized retrieval results. + if len(keys) > 0 { + ancestors, e := p.chunkRepo.ListChunksByIDOnly(ctx, keys) + if e == nil { + for _, c := range ancestors { + byID[c.ID] = c + if c.ParentChunkID != "" { + parents[c.ParentChunkID] = true + } + } + } + } + keys = nil + for id := range parents { + keys = append(keys, id) + } + if len(keys) > 0 { + siblings, e := p.chunkRepo.ListChunksByParentIDsOnly(ctx, keys) + if e == nil { + for _, c := range siblings { + byID[c.ID] = c + } + } + } + // Fetch actual image children too: enrichment may have placed their OCR in + // a text result, but that OCR needs the image child's own citation identity. + textIDs := []string{} + for _, c := range byID { + if c.ChunkType == types.ChunkTypeText { + textIDs = append(textIDs, c.ID) + } + } + if len(textIDs) > 0 { + images, e := p.chunkRepo.ListChunksByParentIDsOnly(ctx, textIDs) + if e == nil { + for _, c := range images { + if c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption { + byID[c.ID] = c + } + } + } + } + for _, r := range results { + if !r.ContentRewritten { + continue + } + allowedParents := map[string]bool{} + for _, id := range append([]string{r.ID}, r.SubChunkID...) { + if c := byID[id]; c != nil { + allowedParents[c.ParentChunkID] = true + if parent := byID[c.ParentChunkID]; parent != nil && parent.ChunkType == types.ChunkTypeText { + allowedParents[parent.ParentChunkID] = true + } + } + } + candidates := []*types.Chunk{} + for _, c := range byID { + if !c.IsEnabled || + c.KnowledgeID != r.KnowledgeID || + c.KnowledgeBaseID != r.KnowledgeBaseID || + c.Content == "" || + c.ChunkType == types.ChunkTypeParentText { + continue + } + related := c.ID == r.ID || containsID(r.SubChunkID, c.ID) || allowedParents[c.ParentChunkID] + if parent := byID[c.ParentChunkID]; parent != nil && (c.ChunkType == types.ChunkTypeImageOCR || + c.ChunkType == types.ChunkTypeImageCaption) { + related = related || allowedParents[parent.ParentChunkID] + } + if !related { + continue + } + body := searchutil.PruneMarkdownImagesByImageInfo(c.Content, r.ImageInfo) + imageHit := false + if c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption { + imageHit = searchutil.FilterImageInfoByContentURLs(r.Content, c.ImageInfo) != "" + } + if c.ID == r.ID || + containsID(r.SubChunkID, + c.ID) || + imageHit || + (strings.TrimSpace(body) != "" && strings.Contains(r.Content, + body)) { + candidates = append(candidates, c) + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].ChunkIndex != candidates[j].ChunkIndex { + return candidates[i].ChunkIndex < candidates[j].ChunkIndex + } + return candidates[i].ID < candidates[j].ID + }) + for _, c := range candidates { + source := *r + source.ID = c.ID + source.Content = c.Content + source.ChunkIndex = c.ChunkIndex + source.ChunkType = string(c.ChunkType) + source.ParentChunkID = c.ParentChunkID + source.SourceLocators = c.SourceLocators + source.ImageInfo = c.ImageInfo + source.StartAt = c.StartAt + source.EndAt = c.EndAt + source.ContentRevision = c.ContentRevision + source.ContextHeader = c.ContextHeader + source.ChunkMetadata = c.Metadata + source.MatchedContent = "" + source.ContentRewritten = false + source.SubChunkID = nil + source.CitationSources = nil + r.CitationSources = append(r.CitationSources, &source) + } + } +} + +func expandCitationSources(results []*types.SearchResult) []*types.SearchResult { + out := make([]*types.SearchResult, 0, len(results)) + seen := map[string]bool{} + for _, r := range results { + if r == nil { + continue + } + sources := r.CitationSources + if len(sources) == 0 && r.ContentRewritten { + // A failed source lookup must not relabel expanded context as evidence. + continue + } + if len(sources) == 0 { + sources = []*types.SearchResult{r} + } + for _, s := range sources { + if !seen[s.ID] { + seen[s.ID] = true + out = append(out, s) + } + } + } + return out +} diff --git a/internal/application/service/chat_pipeline/citation_sources_test.go b/internal/application/service/chat_pipeline/citation_sources_test.go new file mode 100644 index 00000000..52c0d9b0 --- /dev/null +++ b/internal/application/service/chat_pipeline/citation_sources_test.go @@ -0,0 +1,138 @@ +package chatpipeline + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestExpandedContextHasSeparateCitationBodies(t *testing.T) { + a := &types.Chunk{ + ID: "a", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + Content: "Opening greeting.", + ParentChunkID: "p", + ChunkType: types.ChunkTypeText, + IsEnabled: true, + } + b := &types.Chunk{ + ID: "b", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + Content: "Exact later quotation.", + ParentChunkID: "p", + ChunkType: types.ChunkTypeText, + IsEnabled: true, + ChunkIndex: 1, + } + disabled := &types.Chunk{ + ID: "disabled", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + Content: "Exact later quotation.", + ParentChunkID: "p", + ChunkType: types.ChunkTypeText, + } + foreign := &types.Chunk{ + ID: "foreign", + KnowledgeID: "other", + KnowledgeBaseID: "kb", + Content: "Opening greeting.", + IsEnabled: true, + } + repo := &expandChunkRepo{ + chunks: map[string]*types.Chunk{ + "a": a, + "p": { + ID: "p", + ChunkType: types.ChunkTypeParentText, + }, + }, + children: map[string][]*types.Chunk{"p": { + a, + b, + disabled, + foreign, + }}, + } + p := &PluginMerge{chunkRepo: repo} + merged := &types.SearchResult{ + ID: "a", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + Content: "Opening greeting.\n\nExact later quotation.", + ContentRewritten: true, + } + p.attachCitationSources(context.Background(), []*types.SearchResult{merged}) + got := expandCitationSources([]*types.SearchResult{merged, merged}) + require.Len(t, got, 2) + require.Equal(t, "a", got[0].ID) + require.Equal(t, a.Content, got[0].Content) + require.Equal(t, "b", got[1].ID) + require.Equal(t, b.Content, got[1].Content) + require.Empty(t, got[0].SubChunkID) +} + +func TestExpandedEvidenceFailsClosedAndPreservesPlainSources(t *testing.T) { + got := expandCitationSources([]*types.SearchResult{ + { + ID: "expanded", + Content: "parent text under child ID", + ContentRewritten: true, + }, + { + ID: "plain", + Content: "actual source", + }, + }) + require.Len(t, got, 1) + require.Equal(t, "plain", got[0].ID) +} + +func TestModelHandlesReferenceTheActualExpandedChild(t *testing.T) { + a := &types.SearchResult{ + ID: "opening", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + KnowledgeTitle: "Report", + Content: "Opening greeting.", + } + b := &types.SearchResult{ + ID: "later", + KnowledgeID: "doc", + KnowledgeBaseID: "kb", + KnowledgeTitle: "Report", + ChunkIndex: 1, + Content: "Exact later quotation.", + } + rendered := "old expanded context" + manage := &types.ChatManage{ + PipelineRequest: types.PipelineRequest{ + Query: "question", + SummaryConfig: types.SummaryConfig{Prompt: "system"}, + }, + PipelineState: types.PipelineState{ + RenderedContexts: rendered, + UserContent: rendered, + MergeResult: []*types.SearchResult{{ + ID: "opening", + ContentRewritten: true, + CitationSources: []*types.SearchResult{ + a, + b, + }, + }}, + }, + } + messages, registry := prepareMessagesWithModelContext(context.Background(), manage) + require.Contains(t, messages[1].Content, ``), `chunk_id="later"`) + require.Contains(t, registry.DecodeOutputText(``), `chunk_id="opening"`) + manage.MergeResult[0].CitationSources = nil + messages, _ = prepareMessagesWithModelContext(context.Background(), manage) + require.NotContains(t, messages[1].Content, rendered) + require.Contains(t, messages[1].Content, "could not be verified") +} diff --git a/internal/application/service/chat_pipeline/merge.go b/internal/application/service/chat_pipeline/merge.go index 1b370db3..9d131908 100644 --- a/internal/application/service/chat_pipeline/merge.go +++ b/internal/application/service/chat_pipeline/merge.go @@ -92,6 +92,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context, mergedChunks = p.dedup(ctx, "final_dedup", mergedChunks) mergedChunks = removePartialOverlaps(ctx, mergedChunks) + p.attachCitationSources(ctx, mergedChunks) chatManage.MergeResult = mergedChunks return next() } diff --git a/internal/application/service/chat_pipeline/references.go b/internal/application/service/chat_pipeline/references.go index 234149f2..aa69336c 100644 --- a/internal/application/service/chat_pipeline/references.go +++ b/internal/application/service/chat_pipeline/references.go @@ -30,7 +30,7 @@ func prepareMessagesWithModelContext( return messages, registry } - ordered := orderedPipelineReferences(chatManage) + ordered := expandCitationSources(orderedPipelineReferences(chatManage)) knowledgeResults := make([]*types.SearchResult, 0, len(ordered)) knowledgeRows := make([]map[string]interface{}, 0, len(ordered)) webRows := make([]map[string]interface{}, 0) @@ -80,7 +80,8 @@ func prepareMessagesWithModelContext( } modelContexts := strings.Join(contextParts, "\n") if strings.TrimSpace(modelContexts) == "" { - return messages, registry + modelContexts = "Retrieved source bodies could not be verified. Do not cite or infer facts from " + + "unavailable retrieval evidence." } last := len(messages) - 1 diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index bfe770d5..6cae3464 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -568,6 +568,10 @@ func (s *chunkService) UpdateDocumentChunk( chunk.SourceContent = chunk.Content } bodyChanged := newContent != chunk.Content + if bodyChanged { + // Edited evidence no longer maps to the uploaded original. + chunk.SourceLocators = nil + } chunk.Content = newContent chunk.IsEnabled = newEnabled chunk.ContentRevision++ @@ -747,6 +751,7 @@ func (s *chunkService) rebuildParentContent(ctx context.Context, edited *types.C for _, repl := range replacements { baseRunes = append(append(append([]rune{}, baseRunes[:repl.start]...), []rune(repl.content)...), baseRunes[repl.end:]...) } + parent.SourceLocators = nil // Child edits invalidate the original parent evidence. parent.Content = string(baseRunes) for _, conflict := range conflicts { parent.Content = searchutil.JoinChunkContent(parent.Content, conflict.content, "\n\n") diff --git a/internal/application/service/chunk_edit_parent_test.go b/internal/application/service/chunk_edit_parent_test.go index 1d06f7d6..ab0ea2d6 100644 --- a/internal/application/service/chunk_edit_parent_test.go +++ b/internal/application/service/chunk_edit_parent_test.go @@ -157,7 +157,8 @@ func TestUpdateDocumentChunkPreservesGeneratedQuestionsAcrossRevision(t *testing repo := &editableChunkRepo{chunk: &types.Chunk{ ID: "chunk", TenantID: 1, KnowledgeID: "knowledge", KnowledgeBaseID: "kb", Content: "old body", SourceContent: "old body", ContentRevision: 0, - ChunkType: types.ChunkTypeText, IsEnabled: true, IndexStatus: "ready", Metadata: metadataJSON, + SourceLocators: types.SourceLocators{{Type: "pdf", Page: 1, Mapping: "exact", Quote: "old body"}}, + ChunkType: types.ChunkTypeText, IsEnabled: true, IndexStatus: "ready", Metadata: metadataJSON, }} service := &chunkService{ chunkRepository: repo, @@ -182,6 +183,9 @@ func TestUpdateDocumentChunkPreservesGeneratedQuestionsAcrossRevision(t *testing if err != nil { t.Fatalf("update chunk: %v", err) } + if len(updated.SourceLocators) != 0 { + t.Fatal("edited content retained source positions from the original") + } updatedMetadata, err := updated.DocumentMetadata() if err != nil { t.Fatal(err) @@ -219,6 +223,7 @@ func TestRebuildParentContentPreservesConflictingEdits(t *testing.T) { parent: &types.Chunk{ ID: "parent", TenantID: 1, ChunkType: types.ChunkTypeParentText, SourceContent: "abcdefghij", Content: "abcdefghij", StartAt: 0, EndAt: 10, + SourceLocators: types.SourceLocators{{Type: "pdf", Page: 1}}, }, children: []*types.Chunk{ { @@ -240,6 +245,9 @@ func TestRebuildParentContentPreservesConflictingEdits(t *testing.T) { if repo.updated == nil { t.Fatal("parent was not updated") } + if len(repo.updated.SourceLocators) != 0 { + t.Fatal("rebuilt parent retained stale source positions") + } for _, want := range []string{"OLDER EDIT BODY", "NEWER EDIT BODY"} { if !strings.Contains(repo.updated.Content, want) { t.Fatalf("rebuilt parent lost %q: %q", want, repo.updated.Content) diff --git a/internal/application/service/docreader_failure.go b/internal/application/service/docreader_failure.go new file mode 100644 index 00000000..f9e926fa --- /dev/null +++ b/internal/application/service/docreader_failure.go @@ -0,0 +1,30 @@ +package service + +import ( + "context" + "errors" + "net" + "strings" + + werrors "github.com/Tencent/WeKnora/internal/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Keep actionable public messages separate from raw transport diagnostics, +// which may contain internal addresses or credentials and belong in error_detail. +func docReaderFailure(err error) (code, message string) { + if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded || + strings.Contains(err.Error(), "docreader call timeout") { + return werrors.ErrCodeDocReaderTimeout, + "Document parsing service timed out. Check service health and load before retrying." + } + var networkError *net.OpError + if status.Code(err) == codes.Unavailable || errors.As(err, &networkError) { + return werrors.ErrCodeDocReaderUnavailable, + "Cannot connect to the document parsing service, or the connection was interrupted. " + + "Check that DocReader is running and healthy, then retry." + } + return werrors.ErrCodeDocReaderParseFailed, + "Document parsing failed. Check the file format and the DocReader logs for this attempt." +} diff --git a/internal/application/service/docreader_failure_test.go b/internal/application/service/docreader_failure_test.go new file mode 100644 index 00000000..544c937a --- /dev/null +++ b/internal/application/service/docreader_failure_test.go @@ -0,0 +1,52 @@ +package service + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "testing" + + werrors "github.com/Tencent/WeKnora/internal/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestDocReaderFailure(t *testing.T) { + for _, tc := range []struct { + name string + err error + code string + }{ + { + "scanned PDF wrapped connection failure", + fmt.Errorf("anydoc fallback: %w", + fmt.Errorf("read stream: %w", + status.Error(codes.Unavailable, + "private-address secret-token"))), + werrors.ErrCodeDocReaderUnavailable, + }, + { + "HTTP connection failure", + &net.OpError{ + Op: "dial", + Err: errors.New("private-address secret-token"), + }, + werrors.ErrCodeDocReaderUnavailable, + }, + {"context timeout", fmt.Errorf("parse: %w", context.DeadlineExceeded), werrors.ErrCodeDocReaderTimeout}, + {"RPC timeout", status.Error(codes.DeadlineExceeded, "private-address"), werrors.ErrCodeDocReaderTimeout}, + {"parser failure", errors.New("private-address secret-token"), werrors.ErrCodeDocReaderParseFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + code, message := docReaderFailure(tc.err) + if code != tc.code || message == "" { + t.Fatalf("got %q %q, want code %q", code, message, tc.code) + } + if strings.Contains(message, "private-address") || strings.Contains(message, "secret-token") { + t.Fatal("raw transport diagnostics leaked into public message") + } + }) + } +} diff --git a/internal/application/service/image_chunk_owner.go b/internal/application/service/image_chunk_owner.go new file mode 100644 index 00000000..00c3aeb4 --- /dev/null +++ b/internal/application/service/image_chunk_owner.go @@ -0,0 +1,24 @@ +package service + +import ( + "github.com/Tencent/WeKnora/internal/searchutil" + "github.com/Tencent/WeKnora/internal/types" +) + +// A missing association is preferable to attaching evidence to an unrelated +// first paragraph. Exact Markdown targets also avoid URL-prefix collisions. +func imageChunkOwner(url string, chunks []types.ParsedChunk) string { + if url == "" { + return "" + } + owner := "" + for _, c := range chunks { + if c.ChunkID != "" && searchutil.ImageURLsInContent(c.Content)[url] { + if owner != "" && owner != c.ChunkID { + return "" + } + owner = c.ChunkID + } + } + return owner +} diff --git a/internal/application/service/image_chunk_owner_test.go b/internal/application/service/image_chunk_owner_test.go new file mode 100644 index 00000000..33110339 --- /dev/null +++ b/internal/application/service/image_chunk_owner_test.go @@ -0,0 +1,41 @@ +package service + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestImageChunkOwnerUsesExactTargets(t *testing.T) { + chunks := []types.ParsedChunk{ + { + ChunkID: "first", + Content: "unrelated", + }, + { + ChunkID: "prefix", + Content: "![image](resource://abc-more)", + }, + { + ChunkID: "actual", + Content: "![image](resource://abc)", + }, + } + require.Equal(t, "actual", imageChunkOwner("resource://abc", chunks)) + require.Empty(t, imageChunkOwner("resource://missing", chunks)) + require.Empty(t, imageChunkOwner("", chunks)) + require.Empty(t, + imageChunkOwner("resource://abc", + append(chunks, + types.ParsedChunk{ + ChunkID: "duplicate", + Content: "![same](resource://abc)", + }))) + require.Empty(t, + imageChunkOwner("resource://abc", + []types.ParsedChunk{{ + ChunkID: "prose", + Content: "resource://abc", + }})) +} diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go index 6994c9ea..b0e5c7db 100644 --- a/internal/application/service/image_multimodal.go +++ b/internal/application/service/image_multimodal.go @@ -2,6 +2,7 @@ package service import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -317,11 +318,13 @@ func (s *ImageMultimodalService) processImage( logger.Errorf(ctx, "[ImageMultimodal] Skip unreadable image %s: %v", payload.ImageURL, readErr) out["skipped"] = "unreadable_image" out["read_error"] = readErr.Error() - return nil + handleErr = fmt.Errorf("read source image: %w", readErr) + return handleErr } out["image_bytes"] = len(imgBytes) imageInfo := types.ImageInfo{ + SHA256: fmt.Sprintf("%x", sha256.Sum256(imgBytes)), URL: payload.ImageURL, OriginalURL: payload.ImageURL, } diff --git a/internal/application/service/image_multimodal_attrs_test.go b/internal/application/service/image_multimodal_attrs_test.go index 36f78e85..4029a865 100644 --- a/internal/application/service/image_multimodal_attrs_test.go +++ b/internal/application/service/image_multimodal_attrs_test.go @@ -3,7 +3,9 @@ package service import ( "bytes" "context" + "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "strings" @@ -333,6 +335,15 @@ func TestProcessImageObservesAndDescribes(t *testing.T) { if len(repo.created) != 2 { t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created)) } + for _, chunk := range repo.created { + var images []types.ImageInfo + if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 { + t.Fatalf("invalid image info: %v", err) + } + if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) { + t.Fatal("stored image fingerprint does not identify the bytes sent to OCR") + } + } if got := persistedAttrCounts(t, repo.created, "contain.text"); !equalCounts(got, map[string]int{"block": 2}) { t.Errorf("persisted contain.text = %v, want block on both chunks", got) } @@ -553,6 +564,15 @@ func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) { if len(repo.created) != 2 { t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created)) } + for _, chunk := range repo.created { + var images []types.ImageInfo + if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 { + t.Fatalf("invalid image info: %v", err) + } + if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) { + t.Fatal("stored image fingerprint does not identify the bytes sent to OCR") + } + } } // TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption covers the one reply @@ -654,11 +674,19 @@ func TestProcessImageCaptionOCRPipeline(t *testing.T) { if len(repo.created) != 2 { t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created)) } + for _, chunk := range repo.created { + var images []types.ImageInfo + if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 { + t.Fatalf("invalid image info: %v", err) + } + if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) { + t.Fatal("stored image fingerprint does not identify the bytes sent to OCR") + } + } } -// TestProcessImageSkipsUnreadableImage checks that an unreadable object is -// reported and counted rather than failing the task. -func TestProcessImageSkipsUnreadableImage(t *testing.T) { +// Unreadable storage must return its cause so Handle retries before finalizing. +func TestProcessImageReturnsUnreadableImageError(t *testing.T) { t.Parallel() fileSvc := &attrsFileService{err: fmt.Errorf("object is gone")} @@ -678,7 +706,12 @@ func TestProcessImageSkipsUnreadableImage(t *testing.T) { ImageActions: types.DefaultImageActions(), } - out := runProcessImage(t, svc, payload, fake) + out := types.JSONMap{} + err := svc.processImage(context.Background(), payload, fake, types.VLMConfig{}, noopSpanTracker{}, out) + if !errors.Is(err, fileSvc.err) { + t.Fatalf("storage error was swallowed: %v", err) + } + if got := out["skipped"]; got != "unreadable_image" { t.Errorf("skipped = %v, want unreadable_image", got) } diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index ad4cfce6..07e49da3 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3872,6 +3872,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e if convertResult != nil { parsedMarkdown = convertResult.MarkdownContent sourceBlocks = convertResult.SourceBlocks + for i := range sourceBlocks { + sourceBlocks[i].Locator.SourceHash = knowledge.FileHash + } } sanitizeReadResult(convertResult) if convertResult != nil { @@ -4109,16 +4112,11 @@ func (s *knowledgeService) convert( result, err := s.callDocReaderWithTimeout(ctx, reader, req) if err != nil { - // Distinguish DocReader timeout (a knowable user-facing - // failure) from generic read errors so the UI can suggest - // "split this large file" specifically when relevant. - code := werrors.ErrCodeDocReaderParseFailed - if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "docreader call timeout") { - code = werrors.ErrCodeDocReaderTimeout - } + code, message := docReaderFailure(err) + logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err) s.failStage(ctx, knowledge.ID, types.StageDocReader, - code, "document read failed", err) - return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err) + code, message, err) + return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message) } sanitizeReadResult(result) if result.Error != "" { @@ -4292,15 +4290,9 @@ func (s *knowledgeService) enqueueImageMultimodalTasks( for idx, img := range images { // Match image to the ParsedChunk whose content contains the image URL. // ChunkID was populated by processChunks with the real DB UUID. - chunkID := "" - for _, c := range chunks { - if strings.Contains(c.Content, img.ServingURL) { - chunkID = c.ChunkID - break - } - } - if chunkID == "" && len(chunks) > 0 { - chunkID = chunks[0].ChunkID + chunkID := imageChunkOwner(img.ServingURL, chunks) + if chunkID == "" { + logger.Warnf(ctx, "Image has no owning text chunk: knowledge=%s image=%s", knowledge.ID, img.ServingURL) } payload := types.ImageMultimodalPayload{ diff --git a/internal/infrastructure/docparser/grpc_parser.go b/internal/infrastructure/docparser/grpc_parser.go index ac6e6ccd..e78b2ca4 100644 --- a/internal/infrastructure/docparser/grpc_parser.go +++ b/internal/infrastructure/docparser/grpc_parser.go @@ -105,7 +105,7 @@ func (p *GRPCDocumentReader) Close() error { return nil } -var errNotConnected = fmt.Errorf("docreader service not connected") +var errNotConnected = status.Error(codes.Unavailable, "docreader service not connected") func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) { p.mu.RLock() diff --git a/internal/infrastructure/docparser/mineru_layout.go b/internal/infrastructure/docparser/mineru_layout.go index a461e579..169f21eb 100644 --- a/internal/infrastructure/docparser/mineru_layout.go +++ b/internal/infrastructure/docparser/mineru_layout.go @@ -4,6 +4,9 @@ import ( "archive/zip" "bytes" "encoding/json" + "fmt" + "html" + "math" "regexp" "sort" "strings" @@ -20,7 +23,7 @@ const minerUBBoxScale = 1000.0 type minerUContentItem struct { Type string `json:"type"` Text string `json:"text"` - PageIdx int `json:"page_idx"` + PageIdx *int `json:"page_idx"` BBox []float64 `json:"bbox"` TableBody string `json:"table_body"` TableCaption json.RawMessage `json:"table_caption"` @@ -82,8 +85,8 @@ func decodeMinerUContentList(raw []byte) []minerUContentItem { return items } -// minerUContentListFromZip returns the shallowest *content_list.json of a -// MinerU result package, or nil. +// minerUContentListFromZip prefers supported MiddleJson 2.0 geometry and +// falls back to legacy content_list.json for older MinerU packages. func minerUContentListFromZip(zipData []byte) []byte { zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) if err != nil { @@ -92,7 +95,7 @@ func minerUContentListFromZip(zipData []byte) []byte { var candidates []*zip.File for _, f := range zr.File { name := strings.ToLower(f.Name) - if strings.HasSuffix(name, "content_list.json") && !strings.Contains(name, "content_list_v2") { + if strings.HasSuffix(name, "middle_json.json") || strings.HasSuffix(name, "content_list.json") { candidates = append(candidates, f) } } @@ -100,13 +103,34 @@ func minerUContentListFromZip(zipData []byte) []byte { return nil } sort.Slice(candidates, func(i, j int) bool { - return strings.Count(candidates[i].Name, "/") < strings.Count(candidates[j].Name, "/") + mi, + mj := strings.HasSuffix(strings.ToLower(candidates[i].Name), + "middle_json.json"), + strings.HasSuffix(strings.ToLower(candidates[j].Name), + "middle_json.json") + if mi != mj { + return mi + } + di, dj := strings.Count(candidates[i].Name, "/"), strings.Count(candidates[j].Name, "/") + if di != dj { + return di < dj + } + return candidates[i].Name < candidates[j].Name }) - data, err := readZipEntryBytes(candidates[0]) - if err != nil { - return nil + for _, candidate := range candidates { + data, err := readZipEntryBytes(candidate) + if err != nil { + continue + } + if strings.HasSuffix(strings.ToLower(candidate.Name), "middle_json.json") { + var doc minerUMiddle + if json.Unmarshal(data, &doc) != nil || doc.Schema != "docvortex.middle" || doc.Version != "2.0" { + continue + } + } + return data } - return data + return nil } // minerUSourceBlocks aligns MinerU's layout blocks against its markdown so @@ -119,13 +143,26 @@ func minerUSourceBlocks(markdown string, contentList []byte, fileType string) [] if ft != "pdf" && !IsImageFormat(ft) { return nil } + raw := bytes.TrimSpace(contentList) + if len(raw) > 0 && raw[0] == '{' { + return minerUMiddleSourceBlocks(markdown, raw) + } items := decodeMinerUContentList(contentList) if len(items) == 0 { return nil } units := make([]sourceloc.Unit, 0, len(items)) - for _, it := range items { - loc := types.SourceLocator{Type: types.SourceLocatorPDF, Page: it.PageIdx + 1} + for i, it := range items { + if it.PageIdx == nil || *it.PageIdx < 0 { + continue + } + loc := types.SourceLocator{ + Type: types.SourceLocatorPDF, + Page: *it.PageIdx + 1, + SourceID: fmt.Sprintf("mineru:legacy:%d:%d", + *it.PageIdx, + i), + } if bbox, ok := minerUBBox(it.BBox); ok { loc.BBox = bbox } @@ -142,7 +179,7 @@ func minerUBBox(b []float64) ([]float64, bool) { return nil, false } for _, v := range b { - if v < 0 || v > minerUBBoxScale { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > minerUBBoxScale { return nil, false } } @@ -152,3 +189,110 @@ func minerUBBox(b []float64) ([]float64, bool) { } return out, true } + +// MiddleJson 2.0 uses [0,1] normalized boxes (NOT legacy 0..1000). +// Child annotations keep their own geometry. Inline spans have no geometry. +type minerUMiddleBlock struct { + Type string `json:"type"` + Index *int `json:"index"` + BBox []float64 `json:"bbox"` + Content json.RawMessage `json:"content"` +} + +type minerUMiddle struct { + Schema string `json:"schema"` + Version string `json:"schema_version"` + Pages []struct { + PageIdx *int `json:"page_idx"` + Blocks []minerUMiddleBlock `json:"blocks"` + } `json:"pages"` +} + +func minerUMiddleSourceBlocks(markdown string, raw []byte) []types.SourceBlock { + var doc minerUMiddle + if json.Unmarshal(raw, &doc) != nil || doc.Schema != "docvortex.middle" || doc.Version != "2.0" { + return nil + } + var units []sourceloc.Unit + previousPage := -1 + for _, page := range doc.Pages { + if page.PageIdx == nil || *page.PageIdx <= previousPage { + return nil + } + previousPage = *page.PageIdx + previousBlock := -1 + for _, block := range page.Blocks { + if block.Index == nil || *block.Index <= previousBlock { + return nil + } + previousBlock = *block.Index + ref := fmt.Sprintf("mineru:2:%d:%d", *page.PageIdx, *block.Index) + units = append(units, minerUMiddleUnits(block, *page.PageIdx+1, ref, nil)...) + } + } + return sourceloc.Align(markdown, units) +} + +func minerUMiddleUnits(block minerUMiddleBlock, page int, ref string, parentBox []float64) []sourceloc.Unit { + box := block.BBox + if len(box) == 0 { + box = parentBox + } + loc := types.SourceLocator{Type: types.SourceLocatorPDF, Page: page, SourceID: ref} + if normalizedMinerUBox(box) { + loc.BBox = append([]float64(nil), box...) + } + // These containers contain blocks; text and hyperlinks contain inline spans. + switch block.Type { + case "list", "index", "table", "image", "chart", "code": + var children []minerUMiddleBlock + if json.Unmarshal(block.Content, &children) != nil { + return nil + } + var out []sourceloc.Unit + for i, child := range children { + out = append(out, minerUMiddleUnits(child, page, fmt.Sprintf("%s/%d", ref, i), box)...) + } + return out + default: + text := minerUMiddleText(block.Content) + if block.Type == "table_body" || block.Type == "chart_body" { + text = html.UnescapeString(minerUHTMLTagRe.ReplaceAllString(text, " ")) + } + if strings.TrimSpace(text) == "" { + return nil + } + return []sourceloc.Unit{{Text: text, Locator: loc}} + } +} + +func minerUMiddleText(raw json.RawMessage) string { + var text string + if json.Unmarshal(raw, &text) == nil { + return text + } + var spans []struct { + Type string `json:"type"` + Content json.RawMessage `json:"content"` + } + if json.Unmarshal(raw, &spans) != nil { + return "" + } + var out strings.Builder + for _, span := range spans { + out.WriteString(minerUMiddleText(span.Content)) + } + return out.String() +} + +func normalizedMinerUBox(box []float64) bool { + if len(box) != 4 || box[0] >= box[2] || box[1] >= box[3] { + return false + } + for _, v := range box { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > 1 { + return false + } + } + return true +} diff --git a/internal/infrastructure/docparser/mineru_layout_precision_test.go b/internal/infrastructure/docparser/mineru_layout_precision_test.go new file mode 100644 index 00000000..faea0646 --- /dev/null +++ b/internal/infrastructure/docparser/mineru_layout_precision_test.go @@ -0,0 +1,172 @@ +package docparser + +import ( + "archive/zip" + "bytes" + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +// Contract fixture for DocVortex MiddleJson 2.0. Bboxes are normalized +// fractions, page indices are original source indices, and captions are +// child blocks. This is deliberately not a legacy content_list payload. +const middleFixture = `{ + "schema": "docvortex.middle", + "schema_version": "2.0", + "is_full_document": true, + "metadata": { + "file_suffix": "pdf", + "producer": { + "name": "mineru", + "version": "4.0.0" + } + }, + "pages": [ + { + "page_idx": 4, + "blocks": [ + { + "type": "text", + "index": 0, + "bbox": [ + 0.1, + 0.2, + 0.9, + 0.3 + ], + "content": [ + { + "type": "text", + "content": "设备" + }, + { + "type": "hyperlink", + "url": "https://example.org", + "content": [ + { + "type": "text", + "content": "验收合格" + } + ] + } + ] + }, + { + "type": "table", + "index": 1, + "bbox": [ + 0.1, + 0.4, + 0.9, + 0.8 + ], + "content": [ + { + "type": "table_caption", + "bbox": [ + 0.1, + 0.4, + 0.9, + 0.45 + ], + "content": [ + { + "type": "text", + "content": "设备参数表" + } + ] + }, + { + "type": "table_body", + "bbox": [ + 0.1, + 0.46, + 0.9, + 0.8 + ], + "content": "
限值1.5毫米
" + } + ] + } + ] + } + ] +}` + +func TestMinerUMiddleNormalizedGeometryAndNestedBlocks(t *testing.T) { + blocks := minerUSourceBlocks("设备验收合格\n\n设备参数表\n\n|限值|1.5毫米|", []byte(middleFixture), "pdf") + require.Len(t, blocks, 3) + require.Equal(t, 5, blocks[0].Locator.Page) + require.Equal(t, []float64{0.1, 0.2, 0.9, 0.3}, blocks[0].Locator.BBox) + require.Equal(t, "mineru:2:4:0", blocks[0].Locator.SourceID) + require.Equal(t, "exact", blocks[0].Locator.Mapping) + require.Equal(t, []float64{0.1, 0.46, 0.9, 0.8}, blocks[2].Locator.BBox) +} + +func TestMinerUMiddleZIPPreferredToLegacy(t *testing.T) { + var buf bytes.Buffer + z := zip.NewWriter(&buf) + for name, body := range map[string]string{ + "markdown.md": "设备验收合格", + "middle_json.json": middleFixture, + "doc_content_list.json": "[]", + } { + w, err := z.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, z.Close()) + require.JSONEq(t, middleFixture, string(minerUContentListFromZip(buf.Bytes()))) +} + +func TestMinerUMiddleRejectsUnknownSchemaAndInvalidIndices(t *testing.T) { + for _, mutate := range []func(map[string]any){ + func(d map[string]any) { d["schema"] = "docvortex.model" }, + func(d map[string]any) { d["schema_version"] = "9.0" }, + func(d map[string]any) { delete(d["pages"].([]any)[0].(map[string]any), "page_idx") }, + func(d map[string]any) { d["pages"].([]any)[0].(map[string]any)["page_idx"] = -1 }, + } { + var d map[string]any + require.NoError(t, json.Unmarshal([]byte(middleFixture), &d)) + mutate(d) + raw, err := json.Marshal(d) + require.NoError(t, err) + require.Empty(t, minerUSourceBlocks("设备验收合格", raw, "pdf")) + } +} + +func TestMinerULegacyMissingPageIsNotPageOne(t *testing.T) { + require.Empty(t, minerUSourceBlocks("设备验收合格", []byte(`[{"type":"text","text":"设备验收合格"}]`), "pdf")) +} + +func TestMinerUBoxesRejectNonFiniteAndWrongUnits(t *testing.T) { + for _, box := range [][]float64{{0, 0, math.NaN(), 1}, {0, 0, math.Inf(1), 1}, {0, 0, 900, 600}, {.9, 0, .1, 1}} { + require.False(t, normalizedMinerUBox(box)) + } + _, ok := minerUBBox([]float64{0, 0, math.NaN(), 100}) + require.False(t, ok) +} + +func TestMinerULegacyZIPWithUnsupportedMiddleKeepsContentList(t *testing.T) { + var buf bytes.Buffer + z := zip.NewWriter(&buf) + legacy := `[{"type":"text","text":"设备验收合格","page_idx":1,"bbox":[100,200,900,300]}]` + for name, body := range map[string]string{ + "doc_middle_json.json": `{"pdf_info":[]}`, + "doc_content_list.json": legacy, + } { + w, err := z.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, z.Close()) + blocks := minerUSourceBlocks("设备验收合格", minerUContentListFromZip(buf.Bytes()), "pdf") + require.Len(t, blocks, 1) + require.Equal(t, 2, blocks[0].Locator.Page) + require.Equal(t, []float64{.1, .2, .9, .3}, blocks[0].Locator.BBox) +} diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go index 98e69098..28bdfb71 100644 --- a/internal/modelcontext/citations.go +++ b/internal/modelcontext/citations.go @@ -22,6 +22,11 @@ const citationEnabledProtocolPrompt = ` - Source citations are enabled for this answer. Cite a knowledge chunk with exactly and a web page with exactly . - Cite only cN/wN handles backed by tool results for the current task, and only when that source supports the adjacent claim. Never cite dN/bN. +- Each cN covers only its own supplied text or image evidence, not every chunk of the same document. + For a direct quotation, verify that the quoted words occur in the cited cN. Use the later chunk's handle + when the words occur there; never reuse a document's first handle for unrelated passages. +- OCR watermarks, timestamps and image captions are observations of an image. Do not turn a watermark slogan + into a verified service commitment or a caption into a factual document statement without supporting text. - Handles in historical answers, tool arguments, or the bound knowledge-base directory are for navigation, not current evidence. Retrieve the relevant source before citing it. - MCP results are external sources. Use the wN handle for the matching URL in the system-provided diff --git a/internal/sourceloc/align.go b/internal/sourceloc/align.go index e95c7395..f8e09f8c 100644 --- a/internal/sourceloc/align.go +++ b/internal/sourceloc/align.go @@ -1,6 +1,7 @@ package sourceloc import ( + "html" "sort" "strings" "unicode" @@ -16,69 +17,95 @@ type Unit struct { Locator types.SourceLocator } -const ( - // alignKeyRunes is how much of a unit's normalized text is searched for. - alignKeyRunes = 32 - // alignMinRunes skips units too short to place reliably. - alignMinRunes = 2 - // alignWindowRunes bounds how far past the cursor a unit is searched, so - // a unit the parser dropped cannot drag the cursor across the document. - alignWindowRunes = 20000 -) +// Skip units too short to place reliably. +const alignMinRunes = 2 -// Align places units, which are in document order, into markdown and returns -// one block per placed unit. A block runs from where its unit was found to -// where the next placed unit starts, so the blocks tile the markdown from the -// first placed unit on. Units are matched on letters and digits only, which -// makes the match indifferent to Markdown syntax, whitespace and -// punctuation the parser added or dropped. +// Align maps complete source units, never extending a hit through unmatched +// content. Unknown gaps must stay unknown: assigning them to the preceding +// block can point a citation at an entirely different page. func Align(markdown string, units []Unit) []types.SourceBlock { norm := normalizeMarkdown(markdown) - if norm.text == "" || len(units) == 0 { - return nil + runes := []rune(markdown) + var blocks []types.SourceBlock + cursor, floor := 0, 0 + counts := make(map[string]int) + matches := make(map[string][]int) + for _, u := range units { + counts[normalizeKey(u.Text)]++ } - type hit struct { - pos int // rune offset in markdown - unit int - } - var hits []hit - cursor := 0 // byte offset into norm.text - for i, u := range units { + for _, u := range units { key := normalizeKey(u.Text) if utf8.RuneCountInString(key) < alignMinRunes { continue } - at, width := norm.find(key, cursor) - if at < 0 { + // Repeated source text is safe only when every occurrence is accounted + // for by a source unit in reading order (not a TOC or an omitted block). + positions, known := matches[key] + if !known { + positions = normalizedMatches(norm.text, key) + matches[key] = positions + } + if len(positions) != counts[key] { continue } - hits = append(hits, hit{pos: norm.pos[at], unit: i}) - cursor = at + width - } - if len(hits) == 0 { - return nil - } - runes := []rune(markdown) - prev := 0 - for i := range hits { - hits[i].pos = lineLeadStart(runes, hits[i].pos, prev) - prev = hits[i].pos - } - total := len(runes) - blocks := make([]types.SourceBlock, 0, len(hits)) - for i, h := range hits { - end := total - if i+1 < len(hits) { - end = hits[i+1].pos - } - if end <= h.pos { + matchIndex := sort.SearchInts(positions, cursor) + if matchIndex == len(positions) { continue } - blocks = append(blocks, types.SourceBlock{Start: h.pos, End: end, Locator: units[h.unit].Locator}) + at := positions[matchIndex] + endByte := at + len(key) + start := lineLeadStart(runes, norm.pos[at], floor) + end := norm.end[endByte-1] + // Include trailing markup/URLs on the matched line and blank lines, + // but stop before the next visible text's line. This also preserves + // an image's destination without swallowing another paragraph. + if endByte == len(norm.text) { + end = len(runes) + } else { + next := norm.pos[endByte] + lead := next + for lead > 0 && runes[lead-1] != '\n' { + lead-- + } + if lead >= end { + end = lead + } + } + loc := u.Locator + loc.Mapping = "exact" + blocks = append(blocks, types.SourceBlock{Start: start, End: end, Locator: loc}) + cursor, floor = endByte, end } return blocks } +// normalizedMatches rejects a number embedded in a different numeric value. +func normalizedMatches(text, key string) []int { + var out []int + if key == "" { + return out + } + first, _ := utf8.DecodeRuneInString(key) + last, _ := utf8.DecodeLastRuneInString(key) + for from := 0; from <= len(text)-len(key); { + i := strings.Index(text[from:], key) + if i < 0 { + break + } + at := from + i + before, _ := utf8.DecodeLastRuneInString(text[:at]) + after, _ := utf8.DecodeRuneInString(text[at+len(key):]) + left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before)) + right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after)) + if !left && !right { + out = append(out, at) + } + _, size := utf8.DecodeRuneInString(text[at:]) + from = at + size + } + return out +} + // lineLeadStart moves pos back to the start of its line when everything // before it on the line is markup or numbering ("## ", "| ", "1. ", "(2) "), // which belongs to the unit but was not part of its text. It never crosses @@ -95,45 +122,13 @@ func lineLeadStart(runes []rune, pos, floor int) int { return start } -// normalized is the letters-and-digits projection of a text: text holds the +// normalized retains letters, digits and numeric symbols: text holds the // kept runes, pos[b] the rune offset in the source of the rune starting at // byte b of text. type normalized struct { text string pos []int -} - -// find searches key (already normalized) at or after byte offset from, -// within the search window. It tries the key's head first and falls back to -// a slice from its middle, so a unit whose opening differs (a list number -// the parser rendered, a leading image) still lands. It returns the byte -// offset of the match and how far the cursor should advance. -func (n normalized) find(key string, from int) (int, int) { - limit := len(n.text) - if w := from + alignWindowRunes*3; w < limit { - limit = w - } - window := n.text[from:limit] - head := firstRunes(key, alignKeyRunes) - if idx := strings.Index(window, head); idx >= 0 { - return from + idx, len(head) - } - runes := []rune(key) - if len(runes) < alignKeyRunes*2 { - return -1, 0 - } - mid := string(runes[len(runes)/2 : len(runes)/2+alignKeyRunes]) - if idx := strings.Index(window, mid); idx >= 0 { - // Aim at where the unit should begin, not at its middle. - back := len(string(runes[:len(runes)/2])) - start := from + idx - back - start = max(start, from) - for start < from+idx && !utf8.RuneStart(n.text[start]) { - start++ - } - return start, idx + len(mid) - (start - from) - } - return -1, 0 + end []int } // normalizeMarkdown projects markdown onto its letters and digits, skipping @@ -143,6 +138,15 @@ func normalizeMarkdown(markdown string) normalized { var b strings.Builder b.Grow(len(markdown)) pos := make([]int, 0, len(markdown)) + ends := make([]int, 0, len(markdown)) + emit := func(r rune, start, end int) { + before := b.Len() + b.WriteRune(r) + for k := before; k < b.Len(); k++ { + pos = append(pos, start) + ends = append(ends, end) + } + } runes := []rune(markdown) for i := 0; i < len(runes); i++ { r := runes[i] @@ -158,29 +162,77 @@ func normalizeMarkdown(markdown string) normalized { continue } } - if nr, ok := foldRune(r); ok { - start := b.Len() - b.WriteRune(nr) - for k := start; k < b.Len(); k++ { - pos = append(pos, i) + if r == '&' { + if end := indexRune(runes, ';', i+1, 32); end > 0 { + raw := string(runes[i : end+1]) + if decoded := html.UnescapeString(raw); decoded != raw { + context := []rune{' '} + if i > 0 { + context[0] = runes[i-1] + } + context = append(context, []rune(decoded)...) + context = append(context, ' ') + if end+1 < len(runes) { + context[len(context)-1] = runes[end+1] + } + for j := 1; j+1 < len(context); j++ { + if nr, ok := matchRune(context, j); ok { + emit(nr, i, end+1) + } + } + i = end + continue + } } } + if nr, ok := matchRune(runes, i); ok { + emit(nr, i, i+1) + } } - return normalized{text: b.String(), pos: pos} + return normalized{text: b.String(), pos: pos, end: ends} } // normalizeKey projects plain text onto its letters and digits. func normalizeKey(text string) string { var b strings.Builder - b.Grow(len(text)) - for _, r := range text { - if nr, ok := foldRune(r); ok { - b.WriteRune(nr) + runes := []rune(text) + for i := range runes { + if r, ok := matchRune(runes, i); ok { + b.WriteRune(r) } } return b.String() } +// Preserve distinctions such as 1.5/15, -5/5, 5%/5 and <5/>5. +func matchRune(rs []rune, i int) (rune, bool) { + r := rs[i] + if r >= 0xFF01 && r <= 0xFF5E { + r -= 0xFEE0 + } + if !strings.ContainsRune(".,:/+−-%‰<>=≤≥≠", r) { + return foldRune(r) + } + before, after := i-1, i+1 + for before >= 0 && (rs[before] == ' ' || rs[before] == '\t') { + before-- + } + for after < len(rs) && (rs[after] == ' ' || rs[after] == '\t') { + after++ + } + prevDigit := before >= 0 && unicode.IsDigit(rs[before]) + nextDigit := after < len(rs) && unicode.IsDigit(rs[after]) + if ((r == '.' || r == ',' || r == '/' || r == ':') && prevDigit && nextDigit) || + ((r == '-' || r == '+' || r == '−') && nextDigit) || + ((r == '%' || r == '‰') && prevDigit) || strings.ContainsRune("<>=≤≥≠", r) { + if r == '−' { + r = '-' + } + return r, true + } + return foldRune(r) +} + // foldRune keeps letters and digits, lower-cased and with full-width ASCII // folded to its half-width form. func foldRune(r rune) (rune, bool) { @@ -212,17 +264,6 @@ func indexRune(runes []rune, r rune, from, limit int) int { return -1 } -func firstRunes(s string, n int) string { - i := 0 - for idx := range s { - if i == n { - return s[:idx] - } - i++ - } - return s -} - // SortBlocks orders blocks by start offset. func SortBlocks(blocks []types.SourceBlock) { sort.SliceStable(blocks, func(i, j int) bool { return blocks[i].Start < blocks[j].Start }) diff --git a/internal/sourceloc/assign.go b/internal/sourceloc/assign.go index 4dba5efe..f81fc430 100644 --- a/internal/sourceloc/assign.go +++ b/internal/sourceloc/assign.go @@ -1,6 +1,7 @@ package sourceloc import ( + "html" "regexp" "sort" "strings" @@ -9,10 +10,6 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// MaxLocatorsPerChunk caps how many locators one chunk (or merged search -// result) carries. -const MaxLocatorsPerChunk = 64 - // Index answers "which source positions does this range of the final // Markdown cover" for the chunker's output. type Index struct { @@ -51,13 +48,18 @@ func (x *Index) Locators(start, end int) types.SourceLocators { // Blocks before i all end at or before start. i := sort.Search(len(x.blocks), func(k int) bool { return x.maxEnd[k] > start }) var out types.SourceLocators + covered, partial := start, false for ; i < len(x.blocks) && x.blocks[i].Start < end; i++ { b := x.blocks[i] lo, hi := max(b.Start, start), min(b.End, end) if hi <= lo { continue } - quote := CleanQuote(string(x.runes[lo:hi])) + if lo > covered && normalizeMarkdown(fullQuote(string(x.runes[covered:lo]))).text != "" { + partial = true + } + covered = max(covered, hi) + quote := fullQuote(string(x.runes[lo:hi])) if quote == "" && b.Locator.Type != types.SourceLocatorPDF { continue } @@ -66,8 +68,13 @@ func (x *Index) Locators(start, end int) types.SourceLocators { loc.Quote = quote out = AppendLocator(out, loc) } - if len(out) > MaxLocatorsPerChunk { - out = out[:MaxLocatorsPerChunk] + if covered < end && normalizeMarkdown(fullQuote(string(x.runes[covered:end]))).text != "" { + partial = true + } + if partial { + for i := range out { + out[i].Partial = true + } } return out } @@ -101,7 +108,11 @@ func AppendLocator(list types.SourceLocators, loc types.SourceLocator) types.Sou } func foldInto(last *types.SourceLocator, loc types.SourceLocator) bool { - if last.Type != loc.Type { + if last.Type != loc.Type || + last.Partial != loc.Partial || + last.Mapping != loc.Mapping || + last.SourceHash != loc.SourceHash || + last.SourceID != loc.SourceID { return false } switch loc.Type { @@ -158,14 +169,19 @@ func MergeLocators(a, b types.SourceLocators) types.SourceLocators { out = append(out, loc) } } - if len(out) > MaxLocatorsPerChunk { - out = out[:MaxLocatorsPerChunk] - } return out } func sameLocator(a, b types.SourceLocator) bool { - if a.Type != b.Type || a.Page != b.Page || a.Block != b.Block || a.Slide != b.Slide || + if a.Partial != b.Partial || + a.Quote != b.Quote || + a.SourceID != b.SourceID || + a.SourceHash != b.SourceHash || + a.Mapping != b.Mapping || + a.Type != b.Type || + a.Page != b.Page || + a.Block != b.Block || + a.Slide != b.Slide || a.Sheet != b.Sheet || a.RowStart != b.RowStart || a.RowEnd != b.RowEnd || a.Start != b.Start || a.End != b.End || a.StartMs != b.StartMs || a.EndMs != b.EndMs || a.Section != b.Section || len(a.BBox) != len(b.BBox) { @@ -190,21 +206,27 @@ var ( // text, collapses whitespace, and caps it at types.SourceLocatorQuoteMax // runes. func CleanQuote(text string) string { + return capRunes(fullQuote(text), types.SourceLocatorQuoteMax) +} + +// fullQuote keeps the complete evidence; truncation can hide a citation near its end. +func fullQuote(text string) string { text = quoteImageRe.ReplaceAllString(text, " ") text = quoteLinkRe.ReplaceAllString(text, "$1") text = quoteTagRe.ReplaceAllString(text, " ") + text = html.UnescapeString(text) text = strings.TrimSpace(quoteSpaceRe.ReplaceAllString(text, " ")) - return capRunes(text, types.SourceLocatorQuoteMax) + return text } func joinQuote(a, b string) string { if a == "" { return b } - if b == "" || utf8.RuneCountInString(a) >= types.SourceLocatorQuoteMax { + if b == "" { return a } - return capRunes(a+" "+b, types.SourceLocatorQuoteMax) + return a + " " + b } func capRunes(s string, n int) string { diff --git a/internal/sourceloc/precision_test.go b/internal/sourceloc/precision_test.go new file mode 100644 index 00000000..76109235 --- /dev/null +++ b/internal/sourceloc/precision_test.go @@ -0,0 +1,177 @@ +package sourceloc + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" +) + +func TestAlignNeverAttributesMissingBlockToPreviousPage(t *testing.T) { + md := "第一段正确文本。\n第二段实际内容没有匹配。\n第三段正确文本。" + units := []Unit{ + {Text: "第一段正确文本", Locator: types.SourceLocator{Type: "pdf", Page: 1}}, + {Text: "第二段OCR错误内容", Locator: types.SourceLocator{Type: "pdf", Page: 2}}, + {Text: "第三段正确文本", Locator: types.SourceLocator{Type: "pdf", Page: 3}}, + } + index := NewIndex(md, Align(md, units)) + at := runeIndex(md, "第二段") + require.Empty(t, index.Locators(at, at+5)) + require.Equal(t, 3, index.Locators(runeIndex(md, "第三段"), len([]rune(md)))[0].Page) +} + +func TestAlignRequiresFullUnitNotCommonPrefix(t *testing.T) { + prefix := strings.Repeat("共同前缀", 12) + require.Empty(t, + Align(prefix+"验收不合格", + []Unit{{ + Text: prefix + "验收合格", + Locator: types.SourceLocator{ + Type: "pdf", + Page: 1, + }, + }})) +} + +func TestAlignPreservesNumericMeaning(t *testing.T) { + for _, pair := range [][2]string{{"数值15毫米", "数值1.5毫米"}, {"温度5度", "温度-5度"}, {"比例5", "比例5%"}, {"值≤5", "值≥5"}} { + require.Empty(t, + Align(pair[0], + []Unit{{ + Text: pair[1], + Locator: types.SourceLocator{ + Type: "pdf", + Page: 1, + }, + }}), + pair) + } +} + +func TestAlignRejectsUnaccountedDuplicate(t *testing.T) { + require.Empty(t, + Align("目录:设备验收合格。\n正文:设备验收合格。", + []Unit{{ + Text: "设备验收合格", + Locator: types.SourceLocator{ + Type: "pdf", + Page: 9, + }, + }})) + units := []Unit{ + { + Text: "相同段落内容", + Locator: types.SourceLocator{ + Type: "pdf", + Page: 1, + }, + }, + { + Text: "相同段落内容", + Locator: types.SourceLocator{ + Type: "pdf", + Page: 2, + }, + }, + } + blocks := Align("相同段落内容\n相同段落内容", units) + require.Len(t, blocks, 2) + require.Equal(t, 2, blocks[1].Locator.Page) +} + +func TestRemapNeverInterpolatesChangedText(t *testing.T) { + old := "第一段\n已通过验收\n末段" + blocks := []types.SourceBlock{{Start: 4, End: 9, Locator: types.SourceLocator{Type: "pdf", Page: 2}}} + require.Empty(t, RemapBlocks(blocks, old, "第一段\n未通过检查\n末段")) +} + +func TestMergeKeepsDistinctExcerptsOfSameRegion(t *testing.T) { + a := types.SourceLocator{Type: "pdf", Page: 2, BBox: []float64{0, 0, 1, 1}, Quote: "第一句"} + b := a + b.Quote = "另一句" + require.Len(t, MergeLocators(types.SourceLocators{a}, types.SourceLocators{a, b}), 2) +} + +func TestLocatorsPreserveLongEvidenceAndAllRegions(t *testing.T) { + text := strings.Repeat("完整原文", 100) + "末尾引用的关键结论" + index := NewIndex(text, + []types.SourceBlock{{ + Start: 0, + End: len([]rune(text)), + Locator: types.SourceLocator{ + Type: "pdf", + Page: 1, + }, + }}) + require.Equal(t, text, index.Locators(0, len([]rune(text)))[0].Quote) + var blocks []types.SourceBlock + for i := 0; i < 80; i++ { + blocks = append(blocks, + types.SourceBlock{ + Start: i, + End: i + 1, + Locator: types.SourceLocator{ + Type: "pdf", + Page: i + 1, + }, + }) + } + locs := NewIndex(strings.Repeat("文", 80), blocks).Locators(0, 80) + require.Len(t, locs, 80) + require.Len(t, MergeLocators(locs[:40], locs[40:]), 80) +} + +func TestFoldingCannotMixRevisionsOrMappingQuality(t *testing.T) { + first := types.SourceLocator{Type: "pdf", Page: 1, Mapping: "exact", SourceHash: "original", Quote: "第一段"} + second := first + second.Mapping = "" + require.Len(t, AppendLocator(types.SourceLocators{first}, second), 2) + second = first + second.SourceHash = "replacement" + require.Len(t, AppendLocator(types.SourceLocators{first}, second), 2) +} + +func TestAlignRejectsPartialNumericValues(t *testing.T) { + for _, pair := range [][2]string{{"增长为5%", "增长为5"}, {"115", "15"}, {"15.5", "15"}, {"-15", "15"}} { + require.Empty(t, + Align(pair[0], []Unit{{Text: pair[1], Locator: types.SourceLocator{Type: "pdf", Page: 1}}}), + pair, + ) + } +} + +func TestChunkWithUnmappedEvidenceIsMarkedPartial(t *testing.T) { + md := "匹配的原文。\n未匹配的关键结论。" + index := NewIndex(md, Align(md, []Unit{{Text: "匹配的原文", Locator: types.SourceLocator{Type: "pdf", Page: 1}}})) + locs := index.Locators(0, len([]rune(md))) + require.Len(t, locs, 1) + require.True(t, locs[0].Partial) + require.False(t, index.Locators(0, 5)[0].Partial) +} + +func TestAlignmentDecodesTableEntitiesWithoutLosingOffsets(t *testing.T) { + md := "
限值<1.5
" + loc := types.SourceLocator{Type: "pdf", Page: 2} + blocks := Align(md, []Unit{{Text: "限值 <1.5", Locator: loc}}) + require.Len(t, blocks, 1) + quotes := NewIndex(md, blocks).Locators(0, len([]rune(md))) + require.False(t, quotes[0].Partial) + require.Equal(t, "限值 <1.5", quotes[0].Quote) + remapped := RemapBlocks(blocks, md, "|限值|<1.5|") + require.Len(t, remapped, 1) + require.Equal(t, 2, remapped[0].Locator.Page) +} + +func TestImagePlaceholderDoesNotInvalidateVerifiedTextLocators(t *testing.T) { + md := "第一段完整的原文。\n\n![descript](resource://image)\n\n第二段完整的原文。" + blocks := Align(md, []Unit{ + {Text: "第一段完整的原文。", Locator: types.SourceLocator{Type: "docx", Block: 1}}, + {Text: "第二段完整的原文。", Locator: types.SourceLocator{Type: "docx", Block: 3}}, + }) + locs := NewIndex(md, blocks).Locators(0, len([]rune(md))) + require.Len(t, locs, 2) + for _, loc := range locs { + require.False(t, loc.Partial) + } +} diff --git a/internal/sourceloc/remap.go b/internal/sourceloc/remap.go index d67ff41a..be5e6a4a 100644 --- a/internal/sourceloc/remap.go +++ b/internal/sourceloc/remap.go @@ -28,7 +28,8 @@ type Remapper struct { pair []int // pair[i] = new line index matched to old line i, or -1 } -// NewRemapper prepares a mapping from oldText to newText. +// NewRemapper prepares a best-effort offset mapping. +// Deprecated: source positions must use RemapBlocks, which verifies full text. func NewRemapper(oldText, newText string) *Remapper { oldKeys, oldStarts, oldLen := splitLines(oldText) newKeys, newStarts, newLen := splitLines(newText) @@ -120,17 +121,18 @@ func RemapBlocks(blocks []types.SourceBlock, oldText, newText string) []types.So if len(blocks) == 0 || oldText == newText { return blocks } - r := NewRemapper(oldText, newText) - out := make([]types.SourceBlock, 0, len(blocks)) + // Re-align full block text after markup/URL rewrites. Proportional + // interpolation can manufacture a location for deleted or changed content. + oldRunes := []rune(oldText) + units := make([]Unit, 0, len(blocks)) for _, b := range blocks { - start, end := r.Map(b.Start), r.Map(b.End) - if end <= start { + if b.Start < 0 || b.End > len(oldRunes) || b.End <= b.Start { continue } - b.Start, b.End = start, end - out = append(out, b) + text := normalizeMarkdown(string(oldRunes[b.Start:b.End])).text + units = append(units, Unit{Text: text, Locator: b.Locator}) } - return out + return Align(newText, units) } // splitLines returns each line's comparison key (without its line break or a diff --git a/internal/sourceloc/text.go b/internal/sourceloc/text.go index 7a04ee56..e40ef2df 100644 --- a/internal/sourceloc/text.go +++ b/internal/sourceloc/text.go @@ -14,7 +14,7 @@ func TextBlocks(content string) []types.SourceBlock { if start >= 0 && end > start { blocks = append(blocks, types.SourceBlock{ Start: start, End: end, - Locator: types.SourceLocator{Type: types.SourceLocatorText, Start: start, End: end}, + Locator: types.SourceLocator{Type: types.SourceLocatorText, Mapping: "exact", Start: start, End: end}, }) } start = -1 diff --git a/internal/types/chunk.go b/internal/types/chunk.go index cfa834cb..719e7ff7 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -84,6 +84,8 @@ func (f ChunkFlags) ToggleFlag(flag ChunkFlags) ChunkFlags { // ImageInfo 表示与 Chunk 关联的图片信息 type ImageInfo struct { + // SHA256 identifies the exact stored image bytes across parser and preview. + SHA256 string `json:"sha256,omitempty"` // 图片URL(COS) URL string `json:"url" gorm:"type:text"` // 原始图片URL diff --git a/internal/types/search.go b/internal/types/search.go index 3d3c0a9a..1ed0d586 100644 --- a/internal/types/search.go +++ b/internal/types/search.go @@ -149,6 +149,8 @@ func (st SearchTargets) ContainsKB(kbID string) bool { // SearchResult represents the search result type SearchResult struct { + // CitationSources retains independently citeable bodies after context expansion. + CitationSources []*SearchResult `json:"-" gorm:"-"` // ID ID string `gorm:"column:id" json:"id"` // Content diff --git a/internal/types/source_locator.go b/internal/types/source_locator.go index 071f365e..b54d4c05 100644 --- a/internal/types/source_locator.go +++ b/internal/types/source_locator.go @@ -26,13 +26,21 @@ const ( SourceLocatorSection = "section" ) -// SourceLocatorQuoteMax caps the quote stored per locator, in runes. +// SourceLocatorQuoteMax caps display snippets, not stored source evidence. const SourceLocatorQuoteMax = 300 // SourceLocator is one position in an original file. Only the fields of its // Type are set; zero values are omitted from JSON and read back as zero. type SourceLocator struct { Type string `json:"type"` + // Mapping records verified text-to-source alignment. Empty means legacy. + Mapping string `json:"mapping,omitempty"` + // Partial means some of the chunk has no source mapping. + Partial bool `json:"partial,omitempty"` + // SourceID identifies a block within the parser result, not a rendered item. + SourceID string `json:"source_id,omitempty"` + // SourceHash binds the locator to the knowledge record’s source revision hash. + SourceHash string `json:"source_hash,omitempty"` // Page is the 1-based PDF page. Page int `json:"page,omitempty"` // BBox is [x0, y0, x1, y1] as fractions of the page width and height, @@ -57,8 +65,8 @@ type SourceLocator struct { Section int `json:"section,omitempty"` // Title names the section or slide when the source has one. Title string `json:"title,omitempty"` - // Quote is the cited text as parsed, capped at SourceLocatorQuoteMax - // runes. Viewers match it inside the rendered original and against the + // Quote is the complete cited overlap as parsed. Viewers match it + // inside the rendered original and against the // answer sentence that cites the chunk. Quote string `json:"quote,omitempty"` }