diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader
index 43ffeb85..f858c058 100644
--- a/docker/Dockerfile.docreader
+++ b/docker/Dockerfile.docreader
@@ -155,5 +155,8 @@ RUN mkdir -p /tmp/docreader
# 暴露 gRPC 端口
EXPOSE 50051
-# 直接运行 Python 服务(日志输出到 stdout/stderr)
-CMD ["uv", "run", "-m", "docreader.main"]
\ No newline at end of file
+# Probe ARM crypto instructions before importing native dependencies. The
+# entrypoint only enables portable OpenSSL implementations after a SIGILL.
+ENTRYPOINT ["/app/.venv/bin/python", "-m", "docreader.container_entrypoint"]
+# Dependencies are installed from uv.lock at build time; do not mutate them at boot.
+CMD ["/app/.venv/bin/python", "-m", "docreader.main"]
diff --git a/docreader/container_entrypoint.py b/docreader/container_entrypoint.py
new file mode 100644
index 00000000..cd0c8569
--- /dev/null
+++ b/docreader/container_entrypoint.py
@@ -0,0 +1,49 @@
+"""Keep ARM virtual machines with incomplete CPU support from crashing at boot.
+
+Probe in a child: SIGILL cannot be caught as a Python exception. Only that
+specific failure permits the OpenSSL portable path; other failures stay fatal.
+OPENSSL_armcap is documented at https://docs.openssl.org/master/man3/OPENSSL_armcap/
+"""
+
+import os
+import platform
+import signal
+import subprocess
+import sys
+
+
+PROBE = """
+import resource
+resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+cipher = AESGCM(bytes(32))
+nonce = bytes(12)
+encrypted = cipher.encrypt(nonce, b'docreader startup probe', b'probe')
+assert cipher.decrypt(nonce, encrypted, b'probe') == b'docreader startup probe'
+"""
+
+
+def runtime_environment():
+ env = os.environ.copy()
+ if platform.machine().lower() not in {"arm64", "aarch64"} or "OPENSSL_armcap" in env:
+ return env
+ command = [sys.executable, "-c", PROBE]
+ result = subprocess.run(command, env=env, capture_output=True, timeout=30)
+ if result.returncode == -signal.SIGILL:
+ env["OPENSSL_armcap"] = "0"
+ result = subprocess.run(command, env=env, capture_output=True, timeout=30)
+ if result.returncode == 0:
+ print("DocReader: ARM OpenSSL probe raised SIGILL; using portable CPU implementations (OPENSSL_armcap=0).", file=sys.stderr, flush=True)
+ if result.returncode != 0:
+ raise RuntimeError(f"DocReader crypto startup probe failed ({result.returncode}): {result.stderr.decode(errors='replace')}")
+ return env
+
+
+def main():
+ if len(sys.argv) < 2:
+ raise SystemExit("DocReader entrypoint requires a command")
+ os.execvpe(sys.argv[1], sys.argv[1:], runtime_environment())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py
index 1b31db72..744a6350 100644
--- a/docreader/parser/pdf_parser.py
+++ b/docreader/parser/pdf_parser.py
@@ -17,6 +17,7 @@ self-sufficient using pypdfium2 + the Go-side OCR that already exists.
"""
import base64
+from collections import Counter
import io
import logging
import os
@@ -647,7 +648,14 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:
n = textpage.count_chars()
if n <= 0:
return [], 0.0
- width, height = page.get_size()
+ # Glyph coordinates are unrotated PDF user space. get_size() reports
+ # rotated dimensions and would discard valid text on cropped/rotated pages.
+ try:
+ crop_left, crop_bottom, crop_right, crop_top = page.get_cropbox()
+ except Exception:
+ width, height = page.get_size()
+ crop_left, crop_bottom, crop_right, crop_top = 0, 0, width, height
+ width = crop_right - crop_left
invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else []
chars: list = []
@@ -662,7 +670,7 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:
x0, x1 = (left, right) if left <= right else (right, left)
y0, y1 = (bottom, top) if bottom <= top else (top, bottom)
if FILTER_HIDDEN_TEXT:
- if x1 < 0 or x0 > width or y1 < 0 or y0 > height:
+ if x1 < crop_left or x0 > crop_right or y1 < crop_bottom or y0 > crop_top:
continue # off-page glyph
if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible):
continue # covered by an invisible text object
@@ -947,6 +955,11 @@ def _should_prefer_plain(plain: str, layout: str) -> bool:
return True
if not plain:
return False
+ # Geometry must never change numeric evidence (1.5 -> 1 5, -5 -> 5).
+ # Compare a multiset so legitimate column reordering is still allowed.
+ numbers = re.compile(r"[+−-]?\d+(?:[.,:/]\d+)*(?:[%‰])?")
+ if Counter(numbers.findall(plain)) != Counter(numbers.findall(layout)):
+ return True
n, single, punct_only = _layout_line_stats(layout)
if n == 0:
return True
@@ -1489,7 +1502,7 @@ class PDFScannedParser(BaseParser):
{
"start": offset,
"end": offset + len(line),
- "locator": {"type": "pdf", "page": i + 1},
+ "locator": {"type": "pdf", "page": i + 1, "mapping": "exact"},
}
)
offset += len(line)
@@ -1694,7 +1707,7 @@ class PDFParser(BaseParser):
return start
for i in range(page_count):
- page_only = {"type": "pdf", "page": i + 1}
+ page_only = {"type": "pdf", "page": i + 1, "mapping": "exact"}
if classes[i] == "scanned":
page_filename = f"{base_name}_page_{i+1}.jpg"
part = f""
diff --git a/docreader/parser/source_locator.py b/docreader/parser/source_locator.py
index 29ef3e9b..cc3c6659 100644
--- a/docreader/parser/source_locator.py
+++ b/docreader/parser/source_locator.py
@@ -11,14 +11,13 @@ the origin at its top-left corner, so viewers can draw it at any zoom.
from __future__ import annotations
import re
+from collections import Counter
import statistics
import unicodedata
from typing import Iterable, List, Optional, Sequence, Tuple
# How far past the cursor a line is searched for, in normalized characters.
_SEARCH_WINDOW = 6000
-# How much of a line's normalized text is searched for.
-_KEY_CHARS = 24
# Lines shorter than this (normalized) are too ambiguous to place.
_MIN_KEY_CHARS = 2
# Lines shorter than this are placed only when their text is unique on the
@@ -54,7 +53,22 @@ def normalize_with_positions(text: str) -> Tuple[str, List[int]]:
out: List[str] = []
pos: List[int] = []
for i, ch in enumerate(text):
+ value = unicodedata.normalize("NFKC", ch)
folded = _fold(ch)
+ if value in ".,:/+−-%‰<>=≤≥≠":
+ before, after = i - 1, i + 1
+ while before >= 0 and text[before] in " \t":
+ before -= 1
+ while after < len(text) and text[after] in " \t":
+ after += 1
+ prev = before >= 0 and text[before].isdigit()
+ next_digit = after < len(text) and text[after].isdigit()
+ semantic = ((value in ".,/:" and prev and next_digit)
+ or (value in "+-−" and next_digit)
+ or (value in "%‰" and prev)
+ or value in "<>=≤≥≠")
+ if semantic:
+ folded = value.replace("−", "-")
for f in folded:
out.append(f)
pos.append(i)
@@ -62,7 +76,7 @@ def normalize_with_positions(text: str) -> Tuple[str, List[int]]:
def normalize(text: str) -> str:
- return "".join(_fold(ch) for ch in text)
+ return normalize_with_positions(text)[0]
def _key_weight(key: str) -> int:
@@ -81,6 +95,23 @@ def _column_cursor(boxes, idx: int, ends: List[Optional[int]]) -> Optional[int]:
return None
+def _numeric_safe_matches(text: str, key: str) -> List[int]:
+ out = []
+ if not key:
+ return out
+ start = 0
+ while (at := text.find(key, start)) >= 0:
+ before = text[at - 1] if at else ""
+ end = at + len(key)
+ after = text[end] if end < len(text) else ""
+ left = key[0].isdigit() and bool(before) and (before.isdigit() or before in ".,:/+−-")
+ right = key[-1].isdigit() and bool(after) and (after.isdigit() or after in ".,:/%‰")
+ if not left and not right:
+ out.append(at)
+ start = at + 1
+ return out
+
+
def locate_lines(
page_text: str,
line_texts: Sequence[str],
@@ -100,10 +131,12 @@ def locate_lines(
hits: List[Optional[int]] = []
ends: List[Optional[int]] = []
cursor = 0
+ counts = Counter(normalize(line) for line in line_texts)
+ occurrences = {key: _numeric_safe_matches(norm, key) for key in counts}
for idx, line in enumerate(line_texts):
- key = normalize(line)[:_KEY_CHARS]
- if len(key) < _MIN_KEY_CHARS or (
- _key_weight(key) < _SHORT_KEY_WEIGHT and norm.count(key) > 1
+ key = normalize(line)
+ if len(key) < _MIN_KEY_CHARS or len(occurrences[key]) != counts[key] or (
+ _key_weight(key) < _SHORT_KEY_WEIGHT and len(occurrences[key]) > 1
):
hits.append(None)
ends.append(None)
@@ -115,11 +148,11 @@ def locate_lines(
cursors.insert(0, column)
at = -1
for start_at in cursors:
- at = norm.find(key, start_at, start_at + _SEARCH_WINDOW + len(key))
+ at = next((hit for hit in occurrences[key] if start_at <= hit <= start_at + _SEARCH_WINDOW), -1)
if at >= 0:
break
if at < 0:
- at = norm.find(key)
+ at = occurrences[key][0] if len(occurrences[key]) == 1 else -1
if at < 0:
hits.append(None)
ends.append(None)
@@ -232,13 +265,12 @@ def page_blocks(
``lines`` are visual lines ``{"text", "bbox"}`` (bbox in PDF points). Lines
are placed in ``page_text``, grouped into paragraphs by geometry, and each
- paragraph becomes a block spanning from its first line to the next
- paragraph, boxed by the union of its lines. Text before the first placed
- line, or a page with no placeable lines, gets a page-only block, so the
- blocks always tile the page's text.
+ paragraph becomes a block spanning only its matched lines, boxed by their
+ union. Unmatched gaps keep page-only provenance and never inherit the
+ previous paragraph's geometry.
"""
end = offset + len(page_text)
- page_only = {"type": "pdf", "page": page_number}
+ page_only = {"type": "pdf", "page": page_number, "mapping": "exact"}
if not page_text:
return []
placed = []
@@ -264,30 +296,49 @@ def page_blocks(
for g in (_line_gap(a[1], b[1]) for a, b in zip(placed, placed[1:]))
if g is not None and g >= 0
]
- usual_gap = statistics.median(gaps) if gaps else line_h * 0.5
+ usual_gap = min(statistics.median(gaps), line_h * 1.5) if gaps else line_h * 0.5
+
+ norm, positions = normalize_with_positions(page_text)
+ def line_end(at, line):
+ key = normalize(line["text"])
+ norm_start = next((i for i, p in enumerate(positions) if p >= at), len(norm))
+ hit = norm.find(key, norm_start)
+ if hit < 0:
+ return at
+ end_at = positions[hit + len(key) - 1] + 1
+ while end_at < len(page_text) and not page_text[end_at].isalnum():
+ if page_text[end_at] == "\n":
+ return end_at + 1
+ end_at += 1
+ return end_at
paragraphs: List[Tuple[int, List[dict]]] = []
for at, ln in placed:
+ ln = {**ln, "_source_end": line_end(at, ln)}
if paragraphs:
prev = paragraphs[-1][1][-1]
col_right = max(p["bbox"][2] for p in paragraphs[-1][1])
col_right = max(col_right, ln["bbox"][2])
- if not _starts_paragraph(prev, ln, line_h, usual_gap, col_right):
+ previous_end = prev["_source_end"]
+ gap_is_empty = not normalize(page_text[previous_end:at])
+ if gap_is_empty and not _starts_paragraph(prev, ln, line_h, usual_gap, col_right):
paragraphs[-1][1].append(ln)
continue
paragraphs.append((at, [ln]))
blocks: List[dict] = []
- first_at = paragraphs[0][0]
- if first_at > 0 and page_text[:first_at].strip():
- blocks.append({"start": offset, "end": offset + first_at, "locator": dict(page_only)})
- for i, (at, members) in enumerate(paragraphs):
- stop = paragraphs[i + 1][0] if i + 1 < len(paragraphs) else len(page_text)
- if i == 0:
- at = 0 if not blocks else at
+ cursor = 0
+ for at, members in paragraphs:
+ stop = members[-1]["_source_end"]
if stop <= at:
continue
+ if at > cursor and page_text[cursor:at].strip():
+ blocks.append({"start": offset + cursor, "end": offset + at, "locator": dict(page_only)})
locator = dict(page_only)
locator["bbox"] = geometry.bbox(_union(m["bbox"] for m in members))
+ locator["source_id"] = f"pdf:{page_number}:{at}:{stop}"
blocks.append({"start": offset + at, "end": offset + stop, "locator": locator})
+ cursor = stop
+ if cursor < len(page_text) and page_text[cursor:].strip():
+ blocks.append({"start": offset + cursor, "end": end, "locator": dict(page_only)})
return blocks
diff --git a/docreader/tests/generate_source_location_browser_fixtures.py b/docreader/tests/generate_source_location_browser_fixtures.py
new file mode 100644
index 00000000..a5365034
--- /dev/null
+++ b/docreader/tests/generate_source_location_browser_fixtures.py
@@ -0,0 +1,41 @@
+"""Regenerate browser fixtures with the actual builtin PDF parser.
+
+Run from the repository root:
+ docreader/.venv/bin/python -m docreader.tests.generate_source_location_browser_fixtures
+"""
+import base64
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+from docreader.parser.pdf_parser import PDFParser
+from docreader.tests.test_source_locator_pdf import pdf_fixture
+
+
+def main():
+ cases = [{"name": "PDFium lower paragraph with decimal evidence", "page": 2,
+ "needle": "Second page lower evidence", "pages": [
+ {"lines": [("First page has independent original evidence.", 60, 700)]},
+ {"lines": [("Second page upper paragraph is unrelated.", 60, 700),
+ ("Second page lower evidence: the limit is 1.5 mm.", 60, 400)]}]}]
+ for rotation in (0, 90, 180, 270):
+ cases.append({"name": f"PDFium CropBox and rotation {rotation}", "page": 1,
+ "needle": "Cropped and rotated", "pages": [
+ {"rotation": rotation, "crop": "/CropBox [100 100 500 700]",
+ "lines": [("Cropped and rotated source evidence.", 150, 600)]}]})
+ output = []
+ for case in cases:
+ data = pdf_fixture(case["pages"])
+ with patch("docreader.parser.pdf_parser.SOURCE_BOXES", True):
+ doc = PDFParser(file_name="fixture.pdf", pdf_force_scanned=False).parse_into_text(data)
+ at = doc.content.index(case["needle"])
+ block = next(b for b in doc.source_blocks if b["start"] <= at < b["end"])
+ locator = {**block["locator"], "quote": doc.content[block["start"]:block["end"]].strip()}
+ assert locator.get("bbox"), case["name"]
+ output.append({"name": case["name"], "page": case["page"], "pdf": base64.b64encode(data).decode(), "locator": locator})
+ target = Path(__file__).resolve().parents[2] / "frontend/tests/source-locate/parsedFixtures.json"
+ target.write_text(json.dumps(output, ensure_ascii=False, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/docreader/tests/test_container_entrypoint.py b/docreader/tests/test_container_entrypoint.py
new file mode 100644
index 00000000..49a42dac
--- /dev/null
+++ b/docreader/tests/test_container_entrypoint.py
@@ -0,0 +1,64 @@
+import os
+import signal
+import subprocess
+import unittest
+from unittest.mock import patch
+
+from docreader.container_entrypoint import runtime_environment
+
+
+class ContainerEntrypointTest(unittest.TestCase):
+ def setUp(self):
+ env = patch.dict(os.environ, {"PATH": "/app/.venv/bin"}, clear=True)
+ env.start()
+ self.addCleanup(env.stop)
+ machine = patch("docreader.container_entrypoint.platform.machine", return_value="aarch64")
+ machine.start()
+ self.addCleanup(machine.stop)
+
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_healthy_arm_keeps_acceleration(self, run):
+ run.return_value = subprocess.CompletedProcess([], 0, b"", b"")
+ self.assertNotIn("OPENSSL_armcap", runtime_environment())
+ self.assertEqual(run.call_count, 1)
+
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_sigill_retries_and_verifies_portable_crypto(self, run):
+ run.side_effect = [subprocess.CompletedProcess([], -signal.SIGILL, b"", b""), subprocess.CompletedProcess([], 0, b"", b"")]
+ self.assertEqual(runtime_environment()["OPENSSL_armcap"], "0")
+ self.assertEqual(run.call_count, 2)
+ self.assertNotIn("OPENSSL_armcap", os.environ)
+
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_other_failure_is_not_masked(self, run):
+ run.return_value = subprocess.CompletedProcess([], 1, b"", b"missing dependency")
+ with self.assertRaisesRegex(RuntimeError, "missing dependency"):
+ runtime_environment()
+ self.assertEqual(run.call_count, 1)
+
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_failed_portable_probe_stays_fatal(self, run):
+ run.side_effect = [subprocess.CompletedProcess([], -signal.SIGILL, b"", b""), subprocess.CompletedProcess([], 1, b"", b"encryption failed")]
+ with self.assertRaisesRegex(RuntimeError, "encryption failed"):
+ runtime_environment()
+
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_operator_override_is_preserved(self, run):
+ os.environ["OPENSSL_armcap"] = "0x1"
+ self.assertEqual(runtime_environment()["OPENSSL_armcap"], "0x1")
+ run.assert_not_called()
+
+ @patch("docreader.container_entrypoint.platform.machine", return_value="x86_64")
+ @patch("docreader.container_entrypoint.subprocess.run")
+ def test_other_architectures_are_unchanged(self, run, machine):
+ self.assertNotIn("OPENSSL_armcap", runtime_environment())
+ run.assert_not_called()
+
+ @patch("docreader.container_entrypoint.subprocess.run", side_effect=subprocess.TimeoutExpired("probe", 30))
+ def test_probe_has_bounded_startup(self, run):
+ with self.assertRaises(subprocess.TimeoutExpired):
+ runtime_environment()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docreader/tests/test_source_locator.py b/docreader/tests/test_source_locator.py
index f84cf2f4..bbdbf18b 100644
--- a/docreader/tests/test_source_locator.py
+++ b/docreader/tests/test_source_locator.py
@@ -63,7 +63,7 @@ class PageBlocksTest(unittest.TestCase):
def test_page_without_lines_gets_page_block(self):
blocks = page_blocks("文字", 5, 2, [], None)
- self.assertEqual(blocks, [{"start": 5, "end": 7, "locator": {"type": "pdf", "page": 2}}])
+ self.assertEqual(blocks, [{"start": 5, "end": 7, "locator": {"type": "pdf", "page": 2, "mapping": "exact"}}])
class WireTest(unittest.TestCase):
@@ -139,5 +139,41 @@ class SplitLineAtGapsTest(unittest.TestCase):
self.assertEqual(len(_split_line_at_gaps(self._line(chars))), 1)
+
+class PrecisionRegressionTest(unittest.TestCase):
+ def test_common_prefix_cannot_place_a_different_conclusion(self):
+ prefix = "Common opening repeated before the important conclusion "
+ self.assertEqual(locate_lines(prefix + "failed", [prefix + "passed"]), [None])
+
+ def test_numeric_meaning_is_preserved(self):
+ for source, quote in [("limit 15 mm", "limit 1.5 mm"), ("temperature 5 C", "temperature -5 C"), ("rate 5", "rate 5%")]:
+ self.assertEqual(locate_lines(source, [quote]), [None])
+
+ def test_unaccounted_duplicate_does_not_take_first_region(self):
+ quote = "A repeated paragraph with enough text"
+ self.assertEqual(locate_lines(quote + "\n" + quote, [quote]), [None])
+
+ def test_unmatched_text_never_inherits_previous_geometry(self):
+ page = "First matched line.\nUnmatched evidence in between.\nLast matched line."
+ lines = [_ln("First matched line.", 50, 700, 300, 712), _ln("Last matched line.", 50, 668, 300, 680)]
+ blocks = page_blocks(page, 0, 2, lines, PageGeometry((0, 0, 600, 800)))
+ at = page.index("Unmatched")
+ covering = [b for b in blocks if b["start"] <= at < b["end"]]
+ self.assertEqual(len(covering), 1)
+ self.assertNotIn("bbox", covering[0]["locator"])
+ self.assertEqual(covering[0]["locator"]["page"], 2)
+
+ def test_trailing_unmatched_text_has_only_page_provenance(self):
+ page = "Matched paragraph.\nUnrecognized trailing evidence."
+ blocks = page_blocks(page, 0, 7, [_ln("Matched paragraph.", 50, 700, 300, 712)], PageGeometry((0, 0, 600, 800)))
+ self.assertEqual(blocks[0]["end"], page.index("Unrecognized"))
+ self.assertNotIn("bbox", blocks[-1]["locator"])
+
+ def test_wire_preserves_verified_geometry_and_identity(self):
+ from docreader.source_wire import source_blocks_to_proto
+ locator = {"type": "pdf", "page": 501, "mapping": "exact", "source_id": "pdf:501:0:3", "bbox": [.1, .2, .8, .4]}
+ wire = source_blocks_to_proto(Document(content="abc", source_blocks=[{"start": 0, "end": 3, "locator": locator}]))
+ self.assertEqual(json.loads(wire[0].locator_json), locator)
+
if __name__ == "__main__":
unittest.main()
diff --git a/docreader/tests/test_source_locator_pdf.py b/docreader/tests/test_source_locator_pdf.py
new file mode 100644
index 00000000..daa652c9
--- /dev/null
+++ b/docreader/tests/test_source_locator_pdf.py
@@ -0,0 +1,78 @@
+"""Actual PDFium parsing through source blocks and the protobuf JSON boundary."""
+import json
+import unittest
+from unittest.mock import patch
+
+from docreader.parser.pdf_parser import PDFParser
+from docreader.source_wire import source_blocks_to_proto
+from docreader.tests.test_pdf_embedded_images import _pdf_from_objects
+
+
+def pdf_fixture(pages):
+ objects = [
+ b"<< /Type /Catalog /Pages 2 0 R >>",
+ ("<< /Type /Pages /Count %d /Kids [%s] >>" % (
+ len(pages), " ".join(f"{4+i*2} 0 R" for i in range(len(pages)))
+ )).encode(),
+ b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ ]
+ for i, page in enumerate(pages):
+ stream = "\n".join(f"BT /F1 14 Tf {x} {y} Td ({text}) Tj ET" for text, x, y in page["lines"]).encode()
+ objects.extend([
+ (f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate {page.get('rotation', 0)} "
+ f"{page.get('crop', '')} /Resources << /Font << /F1 3 0 R >> >> /Contents {5+i*2} 0 R >>").encode(),
+ f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
+ ])
+ return _pdf_from_objects(objects)
+
+
+class NativePDFSourceTest(unittest.TestCase):
+ def parse(self, pages):
+ with patch("docreader.parser.pdf_parser.SOURCE_BOXES", True):
+ doc = PDFParser(file_name="source-regression.pdf", file_type="pdf", pdf_force_scanned=False).parse_into_text(pdf_fixture(pages))
+ self.assertFalse(doc.metadata.get("scanned_page_count"))
+ return doc
+
+ def test_page_and_region_survive_actual_parser_and_wire(self):
+ doc = self.parse([
+ {"lines": [("First page has independent original evidence.", 60, 700)]},
+ {"lines": [("Second page upper paragraph is unrelated.", 60, 700),
+ ("Second page lower evidence: the limit is 1.5 mm.", 60, 400)]},
+ ])
+ at = doc.content.index("Second page lower evidence")
+ source = [b for b in doc.source_blocks if b["start"] <= at < b["end"]]
+ self.assertEqual(len(source), 1)
+ loc = source[0]["locator"]
+ self.assertEqual(loc["page"], 2)
+ self.assertEqual(loc["mapping"], "exact")
+ self.assertGreater(loc["bbox"][1], .45)
+ self.assertLess(loc["bbox"][3], .55)
+ self.assertIn("1.5 mm", doc.content[source[0]["start"]:source[0]["end"]])
+ self.assertIn(loc, [json.loads(b.locator_json) for b in source_blocks_to_proto(doc)])
+
+ def test_cropbox_and_rotation_use_displayed_page_coordinates(self):
+ for rotation in (0, 90, 180, 270):
+ with self.subTest(rotation=rotation):
+ doc = self.parse([{"rotation": rotation, "crop": "/CropBox [100 100 500 700]",
+ "lines": [("Cropped and rotated source evidence.", 150, 600)]}])
+ loc = next(b["locator"] for b in doc.source_blocks if b["locator"].get("bbox"))
+ x0, y0, x1, y1 = loc["bbox"]
+ self.assertTrue(0 <= x0 < x1 <= 1 and 0 <= y0 < y1 <= 1)
+ if rotation == 0: self.assertLess(y1, .2)
+ elif rotation == 90: self.assertGreater(x0, .8)
+ elif rotation == 180: self.assertGreater(y0, .8)
+ else: self.assertLess(x1, .2)
+
+ def test_identical_paragraphs_keep_distinct_vertical_regions(self):
+ text = "Repeated paragraph with identical original evidence."
+ doc = self.parse([{"lines": [(text, 60, 700), (text, 60, 400)]}])
+ boxed = [b for b in doc.source_blocks if b["locator"].get("bbox")]
+ self.assertEqual(len(boxed), 2)
+ self.assertNotEqual(boxed[0]["locator"]["source_id"], boxed[1]["locator"]["source_id"])
+ self.assertLess(boxed[0]["locator"]["bbox"][1], .2)
+ self.assertGreater(boxed[1]["locator"]["bbox"][1], .45)
+ self.assertIn(text.rstrip('.'), doc.content[boxed[1]["start"]:boxed[1]["end"]])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/frontend/src/components/ChatReferenceSourceView.vue b/frontend/src/components/ChatReferenceSourceView.vue
index 38c2931b..4e3a5bbc 100644
--- a/frontend/src/components/ChatReferenceSourceView.vue
+++ b/frontend/src/components/ChatReferenceSourceView.vue
@@ -58,6 +58,8 @@
+
+
+
+ Source location: document rendering regression
+ {{ finished ? `${results.filter((r) => r.startsWith('PASS')).length}/${results.length} passed` : 'Running…' }}
+ Case results
+ done?.(r)" /> done?.(r)" />
+
+
+
+
diff --git a/frontend/tests/source-locate/README.md b/frontend/tests/source-locate/README.md
new file mode 100644
index 00000000..ac02ee40
--- /dev/null
+++ b/frontend/tests/source-locate/README.md
@@ -0,0 +1,60 @@
+# Source location rendering regressions
+
+From `frontend`, run `npm run dev -- --host 127.0.0.1 --port 5199`, then open
+`http://127.0.0.1:5199/tests/source-locate/index.html` in a browser.
+The page runs 52 cases against the actual Vue previews, PDF.js, docx-preview,
+SheetJS and EPUB renderer. Each result is displayed as PASS/FAIL. The page is
+outside the production entry point and is not included in the app build.
+
+The five `parsedFixtures.json` cases are generated by the real Python PDFium
+parser. Regenerate from the repository root after changing parsing or geometry:
+
+```sh
+docreader/.venv/bin/python -m docreader.tests.generate_source_location_browser_fixtures
+```
+
+The fixtures contain only generated test text. No user documents or remote
+parser services are needed. Assertions check page identity, ambiguity rejection,
+nonempty finite geometry, region containment, all fragments of cross-page
+citations, stale evidence, and rapid document destruction/recreation.
+
+The pure resolver's separate Node tests cover page 501, cancellation, partial
+mapping, numeric meaning, and DOMRect prototype properties. Python tests cover
+native PDF parsing, crop/rotation, duplicate paragraphs and protobuf transport.
+
+## Private local replay
+
+Open the same page with `?local=1` to replay `local/manifest.json`. The `local/`
+directory ignores all fixture files, so uploaded documents and query logs are
+never added to Git. Each manifest entry provides `name`, `file`, `ext`,
+`request` (a `SourceLocateRequest`), expected `precise`, and expected PDF
+`pages`, and optional `firstPage` for the initial navigation target. Files are served only by the local development server. The replay
+checks actual highlighted pages and precision status, not merely whether a
+preview opened. Keep real user files out of `parsedFixtures.json`.
+
+The generated DOCX cases also check an OCR image bracketed by unique source
+paragraphs, missing/duplicate image contexts, images at chunk boundaries,
+and a valid sentence within a partially mapped chunk.
+
+PDF cases include column-serialized tables with repeated course names and
+paired row values, and image-heavy cross-page instructions with corrupted font
+arrows. Separate resolver tests reject wrong grades, mismatched rows, duplicate
+tables, and numeric changes. Partial highlights never claim full chunk coverage.
+
+DOCX regressions also verify exact image-byte identity, ambiguous duplicate images,
+and private replay assertions for the exact highlighted excerpt and image digest.
+
+Markdown regressions render cited raw Markdown using the same sanitized renderer
+as the preview. They cover fenced language names, literal tags inside code,
+exact commands, paragraph-level paraphrases, multi-paragraph citations, ambiguous
+duplicate sources, missing source scopes, and changed numeric values. DOM anchor
+cases verify that a citation keeps all preceding sentences since the previous
+citation. Private Markdown fixtures must provide `request.sourceMarkdown` before
+markup stripping; `highlightText` asserts the exact highlighted source paragraphs.
+
+Chunk-boundary cases assert complete original paragraph context when a child
+ends or starts mid-paragraph. Candidate paragraphs must intersect a uniquely
+verified source region with at least eight normalized characters. Duplicate
+source scopes, modified values and unrelated adjacent paragraphs cannot use this
+completion path. A repeated paragraph keeps the occurrence established by the
+full source scope. Exact explicit quotations still retain their narrow range.
diff --git a/frontend/tests/source-locate/index.html b/frontend/tests/source-locate/index.html
new file mode 100644
index 00000000..4794f478
--- /dev/null
+++ b/frontend/tests/source-locate/index.html
@@ -0,0 +1,3 @@
+
+
Source location regression tests
+
diff --git a/frontend/tests/source-locate/local/.gitignore b/frontend/tests/source-locate/local/.gitignore
new file mode 100644
index 00000000..d6b7ef32
--- /dev/null
+++ b/frontend/tests/source-locate/local/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/frontend/tests/source-locate/main.ts b/frontend/tests/source-locate/main.ts
new file mode 100644
index 00000000..1b16c5eb
--- /dev/null
+++ b/frontend/tests/source-locate/main.ts
@@ -0,0 +1,9 @@
+import { createApp } from 'vue'
+import { createI18n } from 'vue-i18n'
+import { Button, Loading } from 'tdesign-vue-next'
+import { Icon } from 'tdesign-icons-vue-next'
+import Harness from './Harness.vue'
+createApp(Harness).component('t-button', Button).component('t-loading', Loading).component('t-icon', Icon).use(createI18n({ legacy: false, locale: 'en', messages: { en: {
+ preview: { zoomOut: 'Zoom out', zoomIn: 'Zoom in', loadFailed: 'Load failed' },
+ chat: { referenceSourcePrevious: 'Previous location', referenceSourceNext: 'Next location' },
+} } })).mount('#app')
diff --git a/frontend/tests/source-locate/officeFixtures.ts b/frontend/tests/source-locate/officeFixtures.ts
new file mode 100644
index 00000000..ca48d483
--- /dev/null
+++ b/frontend/tests/source-locate/officeFixtures.ts
@@ -0,0 +1,36 @@
+import JSZip from 'jszip'
+import * as XLSX from 'xlsx'
+
+export async function docxFixture(): Promise {
+ const zip = new JSZip()
+ zip.file('[Content_Types].xml', '')
+ zip.file('_rels/.rels', '')
+ zip.file('word/document.xml', 'Unrelated first paragraph.The cited limit is 1.5 millimeters.')
+ return zip.generateAsync({ type: 'blob', mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' })
+}
+
+export function sheetFixture(): Blob {
+ const book = XLSX.utils.book_new()
+ XLSX.utils.book_append_sheet(book, XLSX.utils.aoa_to_sheet([['Equipment', 'Limit'], ['Anchor', '1.5 mm'], ['Cable', '15 mm']]), 'Inspection')
+ return new Blob([XLSX.write(book, { type: 'array', bookType: 'xlsx' })])
+}
+
+export async function epubFixture(): Promise {
+ const zip = new JSZip()
+ zip.file('mimetype', 'application/epub+zip')
+ zip.file('META-INF/container.xml', '')
+ zip.file('OEBPS/content.opf', ' ')
+ zip.file('OEBPS/one.xhtml', 'Repeated cited passage.
')
+ zip.file('OEBPS/two.xhtml', 'Repeated cited passage.
')
+ return zip.generateAsync({ type: 'blob', mimeType: 'application/epub+zip' })
+}
+
+export async function docxImageFixture(count = 1): Promise {
+ const zip = await JSZip.loadAsync(await (await docxFixture()).arrayBuffer())
+ zip.file('word/_rels/document.xml.rels', '')
+ zip.file('word/media/image.png', 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', { base64: true })
+ const drawing = ''
+ const xml = await zip.file('word/document.xml')!.async('string')
+ zip.file('word/document.xml', xml.replace('', drawing.repeat(count) + 'Verified paragraph after the image.'))
+ return zip.generateAsync({ type: 'blob' })
+}
diff --git a/frontend/tests/source-locate/parsedFixtures.json b/frontend/tests/source-locate/parsedFixtures.json
new file mode 100644
index 00000000..7f65b07a
--- /dev/null
+++ b/frontend/tests/source-locate/parsedFixtures.json
@@ -0,0 +1,92 @@
+[
+ {
+ "name": "PDFium lower paragraph with decimal evidence",
+ "page": 2,
+ "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAyIC9LaWRzIFs0IDAgUiA2IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAwICAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSAzIDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjUgMCBvYmoKPDwgL0xlbmd0aCA3NiA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDYwIDcwMCBUZCAoRmlyc3QgcGFnZSBoYXMgaW5kZXBlbmRlbnQgb3JpZ2luYWwgZXZpZGVuY2UuKSBUaiBFVAplbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MDAgODAwXSAvUm90YXRlIDAgIC9SZXNvdXJjZXMgPDwgL0ZvbnQgPDwgL0YxIDMgMCBSID4+ID4+IC9Db250ZW50cyA3IDAgUiA+PgplbmRvYmoKNyAwIG9iago8PCAvTGVuZ3RoIDE1MiA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDYwIDcwMCBUZCAoU2Vjb25kIHBhZ2UgdXBwZXIgcGFyYWdyYXBoIGlzIHVucmVsYXRlZC4pIFRqIEVUCkJUIC9GMSAxNCBUZiA2MCA0MDAgVGQgKFNlY29uZCBwYWdlIGxvd2VyIGV2aWRlbmNlOiB0aGUgbGltaXQgaXMgMS41IG1tLikgVGogRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgOAowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTIxIDAwMDAwIG4gCjAwMDAwMDAxOTEgMDAwMDAgbiAKMDAwMDAwMDMyOCAwMDAwMCBuIAowMDAwMDAwNDU0IDAwMDAwIG4gCjAwMDAwMDA1OTEgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA4IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo3OTQKJSVFT0YK",
+ "locator": {
+ "type": "pdf",
+ "page": 2,
+ "mapping": "exact",
+ "bbox": [
+ 0.1,
+ 0.4842,
+ 0.6071,
+ 0.5037
+ ],
+ "source_id": "pdf:2:42:90",
+ "quote": "Second page lower evidence: the limit is 1.5 mm."
+ }
+ },
+ {
+ "name": "PDFium CropBox and rotation 0",
+ "page": 1,
+ "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAwIC9Dcm9wQm94IFsxMDAgMTAwIDUwMCA3MDBdIC9SZXNvdXJjZXMgPDwgL0ZvbnQgPDwgL0YxIDMgMCBSID4+ID4+IC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNSAwIG9iago8PCAvTGVuZ3RoIDY4ID4+CnN0cmVhbQpCVCAvRjEgMTQgVGYgMTUwIDYwMCBUZCAoQ3JvcHBlZCBhbmQgcm90YXRlZCBzb3VyY2UgZXZpZGVuY2UuKSBUaiBFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDE4NSAwMDAwMCBuIAowMDAwMDAwMzQ4IDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKNDY2CiUlRU9GCg==",
+ "locator": {
+ "type": "pdf",
+ "page": 1,
+ "mapping": "exact",
+ "bbox": [
+ 0.125,
+ 0.1456,
+ 0.7203,
+ 0.1716
+ ],
+ "source_id": "pdf:1:0:36",
+ "quote": "Cropped and rotated source evidence."
+ }
+ },
+ {
+ "name": "PDFium CropBox and rotation 90",
+ "page": 1,
+ "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSA5MCAvQ3JvcEJveCBbMTAwIDEwMCA1MDAgNzAwXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSAzIDAgUiA+PiA+PiAvQ29udGVudHMgNSAwIFIgPj4KZW5kb2JqCjUgMCBvYmoKPDwgL0xlbmd0aCA2OCA+PgpzdHJlYW0KQlQgL0YxIDE0IFRmIDE1MCA2MDAgVGQgKENyb3BwZWQgYW5kIHJvdGF0ZWQgc291cmNlIGV2aWRlbmNlLikgVGogRVQKZW5kc3RyZWFtCmVuZG9iagp4cmVmCjAgNgowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMDkgMDAwMDAgbiAKMDAwMDAwMDA1OCAwMDAwMCBuIAowMDAwMDAwMTE1IDAwMDAwIG4gCjAwMDAwMDAxODUgMDAwMDAgbiAKMDAwMDAwMDM0OSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDYgL1Jvb3QgMSAwIFIgPj4Kc3RhcnR4cmVmCjQ2NwolJUVPRgo=",
+ "locator": {
+ "type": "pdf",
+ "page": 1,
+ "mapping": "exact",
+ "bbox": [
+ 0.8284,
+ 0.125,
+ 0.8544,
+ 0.7203
+ ],
+ "source_id": "pdf:1:0:36",
+ "quote": "Cropped and rotated source evidence."
+ }
+ },
+ {
+ "name": "PDFium CropBox and rotation 180",
+ "page": 1,
+ "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAxODAgL0Nyb3BCb3ggWzEwMCAxMDAgNTAwIDcwMF0gL1Jlc291cmNlcyA8PCAvRm9udCA8PCAvRjEgMyAwIFIgPj4gPj4gL0NvbnRlbnRzIDUgMCBSID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNjggPj4Kc3RyZWFtCkJUIC9GMSAxNCBUZiAxNTAgNjAwIFRkIChDcm9wcGVkIGFuZCByb3RhdGVkIHNvdXJjZSBldmlkZW5jZS4pIFRqIEVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMTg1IDAwMDAwIG4gCjAwMDAwMDAzNTAgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0NjgKJSVFT0YK",
+ "locator": {
+ "type": "pdf",
+ "page": 1,
+ "mapping": "exact",
+ "bbox": [
+ 0.2797,
+ 0.8284,
+ 0.875,
+ 0.8544
+ ],
+ "source_id": "pdf:1:0:36",
+ "quote": "Cropped and rotated source evidence."
+ }
+ },
+ {
+ "name": "PDFium CropBox and rotation 270",
+ "page": 1,
+ "pdf": "JVBERi0xLjcKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9Db3VudCAxIC9LaWRzIFs0IDAgUl0gPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+CmVuZG9iago0IDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL01lZGlhQm94IFswIDAgNjAwIDgwMF0gL1JvdGF0ZSAyNzAgL0Nyb3BCb3ggWzEwMCAxMDAgNTAwIDcwMF0gL1Jlc291cmNlcyA8PCAvRm9udCA8PCAvRjEgMyAwIFIgPj4gPj4gL0NvbnRlbnRzIDUgMCBSID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNjggPj4Kc3RyZWFtCkJUIC9GMSAxNCBUZiAxNTAgNjAwIFRkIChDcm9wcGVkIGFuZCByb3RhdGVkIHNvdXJjZSBldmlkZW5jZS4pIFRqIEVUCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMTg1IDAwMDAwIG4gCjAwMDAwMDAzNTAgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0NjgKJSVFT0YK",
+ "locator": {
+ "type": "pdf",
+ "page": 1,
+ "mapping": "exact",
+ "bbox": [
+ 0.1456,
+ 0.2797,
+ 0.1716,
+ 0.875
+ ],
+ "source_id": "pdf:1:0:36",
+ "quote": "Cropped and rotated source evidence."
+ }
+ }
+]
diff --git a/frontend/tests/source-locate/pdfFixture.ts b/frontend/tests/source-locate/pdfFixture.ts
new file mode 100644
index 00000000..fb297a63
--- /dev/null
+++ b/frontend/tests/source-locate/pdfFixture.ts
@@ -0,0 +1,20 @@
+/** Tiny real PDF fixture, generated without a PDF library or network service. */
+export function fixturePdf(pages: Array<{ lines: Array<[string, number, number]>; rotation?: number }>): ArrayBuffer {
+ const objects: string[] = []
+ objects[0] = '<< /Type /Catalog /Pages 2 0 R >>'
+ objects[1] = `<< /Type /Pages /Count ${pages.length} /Kids [${pages.map((_, i) => `${4 + i * 2} 0 R`).join(' ')}] >>`
+ objects[2] = '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'
+ pages.forEach((page, i) => {
+ const stream = page.lines.map(([text, x, y]) => `BT /F1 14 Tf ${x} ${y} Td (${text.replace(/[\\()]/g, '\\$&')}) Tj ET`).join('\n')
+ objects[3 + i * 2] = `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Rotate ${page.rotation || 0} /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + i * 2} 0 R >>`
+ objects[4 + i * 2] = `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`
+ })
+ let pdf = '%PDF-1.4\n'
+ const offsets = [0]
+ objects.forEach((object, i) => { offsets.push(pdf.length); pdf += `${i + 1} 0 obj\n${object}\nendobj\n` })
+ const xref = pdf.length
+ pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`
+ pdf += offsets.slice(1).map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`).join('')
+ pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF`
+ return new TextEncoder().encode(pdf).buffer
+}
diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go
index cf5ada5e..6b6d340c 100644
--- a/internal/application/repository/chunk.go
+++ b/internal/application/repository/chunk.go
@@ -414,6 +414,7 @@ func (r *chunkRepository) SaveChunkRevision(
Updates(map[string]interface{}{
"content": common.CleanInvalidUTF8(chunk.Content),
"source_content": common.CleanInvalidUTF8(chunk.SourceContent),
+ "source_locators": chunk.SourceLocators,
"content_revision": chunk.ContentRevision,
"is_enabled": chunk.IsEnabled,
"metadata": chunk.Metadata,
diff --git a/internal/application/repository/chunk_revision_test.go b/internal/application/repository/chunk_revision_test.go
index c8c518ac..4226ed3b 100644
--- a/internal/application/repository/chunk_revision_test.go
+++ b/internal/application/repository/chunk_revision_test.go
@@ -23,9 +23,20 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) {
chunk := &types.Chunk{
ID: uuid.NewString(), TenantID: 1, KnowledgeBaseID: "kb", KnowledgeID: "knowledge",
Content: "before", SourceContent: "before", ChunkType: types.ChunkTypeText,
+ SourceLocators: types.SourceLocators{{
+ Type: "pdf",
+ Page: 1,
+ Mapping: "exact",
+ SourceID: "block:1",
+ SourceHash: "revision-one",
+ Partial: true,
+ }},
IsEnabled: true, IndexStatus: "ready", CreatedAt: now, UpdatedAt: now,
}
require.NoError(t, repo.CreateChunks(ctx, []*types.Chunk{chunk}))
+ initial, err := repo.GetChunkByID(ctx, 1, chunk.ID)
+ require.NoError(t, err)
+ require.Equal(t, chunk.SourceLocators, initial.SourceLocators)
snapshot := &types.ChunkRevision{
ID: uuid.NewString(), TenantID: 1, KnowledgeBaseID: "kb", KnowledgeID: "knowledge",
@@ -33,12 +44,14 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) {
EditSource: "user", EditedAt: now, CreatedAt: now,
}
chunk.Content = "after"
+ chunk.SourceLocators = nil
chunk.ContentRevision = 1
require.NoError(t, repo.SaveChunkRevision(ctx, chunk, snapshot, 0))
stored, err := repo.GetChunkByID(ctx, 1, chunk.ID)
require.NoError(t, err)
require.Equal(t, "after", stored.Content)
+ require.Empty(t, stored.SourceLocators)
require.Equal(t, 1, stored.ContentRevision)
revisions, err := repo.ListChunkRevisions(ctx, 1, chunk.ID)
require.NoError(t, err)
@@ -55,6 +68,7 @@ func TestSaveChunkRevisionIsAtomicAndOptimistic(t *testing.T) {
stored, err = repo.GetChunkByID(ctx, 1, chunk.ID)
require.NoError(t, err)
require.Equal(t, "after", stored.Content)
+ require.Empty(t, stored.SourceLocators)
count := int64(0)
require.NoError(t, db.Model(&types.ChunkRevision{}).Count(&count).Error)
require.Equal(t, int64(1), count)
diff --git a/internal/application/service/chat_pipeline/citation_sources.go b/internal/application/service/chat_pipeline/citation_sources.go
new file mode 100644
index 00000000..266dfc14
--- /dev/null
+++ b/internal/application/service/chat_pipeline/citation_sources.go
@@ -0,0 +1,192 @@
+package chatpipeline
+
+import (
+ "context"
+ "sort"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/searchutil"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Ranking works on expanded contexts. The model must instead see every source
+// body under its own ID; a parent's neighbouring paragraph is not child evidence.
+func (p *PluginMerge) attachCitationSources(ctx context.Context, results []*types.SearchResult) {
+ if p.chunkRepo == nil {
+ return
+ }
+ ids := map[string]bool{}
+ for _, r := range results {
+ if len(r.SubChunkID) > 0 {
+ r.ContentRewritten = true
+ }
+ if r.ContentRewritten {
+ ids[r.ID] = true
+ for _, id := range r.SubChunkID {
+ ids[id] = true
+ }
+ }
+ }
+ if len(ids) == 0 {
+ return
+ }
+ keys := make([]string, 0, len(ids))
+ for id := range ids {
+ keys = append(keys, id)
+ }
+ chunks, err := p.chunkRepo.ListChunksByIDOnly(ctx, keys)
+ if err != nil {
+ return
+ }
+ byID := map[string]*types.Chunk{}
+ parents := map[string]bool{}
+ for _, c := range chunks {
+ byID[c.ID] = c
+ if c.ParentChunkID != "" {
+ parents[c.ParentChunkID] = true
+ }
+ }
+ keys = nil
+ for id := range parents {
+ keys = append(keys, id)
+ }
+ // Image -> text -> parent_text. Fetch the intermediate text parent before
+ // collecting siblings. All IDs originate in authorized retrieval results.
+ if len(keys) > 0 {
+ ancestors, e := p.chunkRepo.ListChunksByIDOnly(ctx, keys)
+ if e == nil {
+ for _, c := range ancestors {
+ byID[c.ID] = c
+ if c.ParentChunkID != "" {
+ parents[c.ParentChunkID] = true
+ }
+ }
+ }
+ }
+ keys = nil
+ for id := range parents {
+ keys = append(keys, id)
+ }
+ if len(keys) > 0 {
+ siblings, e := p.chunkRepo.ListChunksByParentIDsOnly(ctx, keys)
+ if e == nil {
+ for _, c := range siblings {
+ byID[c.ID] = c
+ }
+ }
+ }
+ // Fetch actual image children too: enrichment may have placed their OCR in
+ // a text result, but that OCR needs the image child's own citation identity.
+ textIDs := []string{}
+ for _, c := range byID {
+ if c.ChunkType == types.ChunkTypeText {
+ textIDs = append(textIDs, c.ID)
+ }
+ }
+ if len(textIDs) > 0 {
+ images, e := p.chunkRepo.ListChunksByParentIDsOnly(ctx, textIDs)
+ if e == nil {
+ for _, c := range images {
+ if c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
+ byID[c.ID] = c
+ }
+ }
+ }
+ }
+ for _, r := range results {
+ if !r.ContentRewritten {
+ continue
+ }
+ allowedParents := map[string]bool{}
+ for _, id := range append([]string{r.ID}, r.SubChunkID...) {
+ if c := byID[id]; c != nil {
+ allowedParents[c.ParentChunkID] = true
+ if parent := byID[c.ParentChunkID]; parent != nil && parent.ChunkType == types.ChunkTypeText {
+ allowedParents[parent.ParentChunkID] = true
+ }
+ }
+ }
+ candidates := []*types.Chunk{}
+ for _, c := range byID {
+ if !c.IsEnabled ||
+ c.KnowledgeID != r.KnowledgeID ||
+ c.KnowledgeBaseID != r.KnowledgeBaseID ||
+ c.Content == "" ||
+ c.ChunkType == types.ChunkTypeParentText {
+ continue
+ }
+ related := c.ID == r.ID || containsID(r.SubChunkID, c.ID) || allowedParents[c.ParentChunkID]
+ if parent := byID[c.ParentChunkID]; parent != nil && (c.ChunkType == types.ChunkTypeImageOCR ||
+ c.ChunkType == types.ChunkTypeImageCaption) {
+ related = related || allowedParents[parent.ParentChunkID]
+ }
+ if !related {
+ continue
+ }
+ body := searchutil.PruneMarkdownImagesByImageInfo(c.Content, r.ImageInfo)
+ imageHit := false
+ if c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
+ imageHit = searchutil.FilterImageInfoByContentURLs(r.Content, c.ImageInfo) != ""
+ }
+ if c.ID == r.ID ||
+ containsID(r.SubChunkID,
+ c.ID) ||
+ imageHit ||
+ (strings.TrimSpace(body) != "" && strings.Contains(r.Content,
+ body)) {
+ candidates = append(candidates, c)
+ }
+ }
+ sort.Slice(candidates, func(i, j int) bool {
+ if candidates[i].ChunkIndex != candidates[j].ChunkIndex {
+ return candidates[i].ChunkIndex < candidates[j].ChunkIndex
+ }
+ return candidates[i].ID < candidates[j].ID
+ })
+ for _, c := range candidates {
+ source := *r
+ source.ID = c.ID
+ source.Content = c.Content
+ source.ChunkIndex = c.ChunkIndex
+ source.ChunkType = string(c.ChunkType)
+ source.ParentChunkID = c.ParentChunkID
+ source.SourceLocators = c.SourceLocators
+ source.ImageInfo = c.ImageInfo
+ source.StartAt = c.StartAt
+ source.EndAt = c.EndAt
+ source.ContentRevision = c.ContentRevision
+ source.ContextHeader = c.ContextHeader
+ source.ChunkMetadata = c.Metadata
+ source.MatchedContent = ""
+ source.ContentRewritten = false
+ source.SubChunkID = nil
+ source.CitationSources = nil
+ r.CitationSources = append(r.CitationSources, &source)
+ }
+ }
+}
+
+func expandCitationSources(results []*types.SearchResult) []*types.SearchResult {
+ out := make([]*types.SearchResult, 0, len(results))
+ seen := map[string]bool{}
+ for _, r := range results {
+ if r == nil {
+ continue
+ }
+ sources := r.CitationSources
+ if len(sources) == 0 && r.ContentRewritten {
+ // A failed source lookup must not relabel expanded context as evidence.
+ continue
+ }
+ if len(sources) == 0 {
+ sources = []*types.SearchResult{r}
+ }
+ for _, s := range sources {
+ if !seen[s.ID] {
+ seen[s.ID] = true
+ out = append(out, s)
+ }
+ }
+ }
+ return out
+}
diff --git a/internal/application/service/chat_pipeline/citation_sources_test.go b/internal/application/service/chat_pipeline/citation_sources_test.go
new file mode 100644
index 00000000..52c0d9b0
--- /dev/null
+++ b/internal/application/service/chat_pipeline/citation_sources_test.go
@@ -0,0 +1,138 @@
+package chatpipeline
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestExpandedContextHasSeparateCitationBodies(t *testing.T) {
+ a := &types.Chunk{
+ ID: "a",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ Content: "Opening greeting.",
+ ParentChunkID: "p",
+ ChunkType: types.ChunkTypeText,
+ IsEnabled: true,
+ }
+ b := &types.Chunk{
+ ID: "b",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ Content: "Exact later quotation.",
+ ParentChunkID: "p",
+ ChunkType: types.ChunkTypeText,
+ IsEnabled: true,
+ ChunkIndex: 1,
+ }
+ disabled := &types.Chunk{
+ ID: "disabled",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ Content: "Exact later quotation.",
+ ParentChunkID: "p",
+ ChunkType: types.ChunkTypeText,
+ }
+ foreign := &types.Chunk{
+ ID: "foreign",
+ KnowledgeID: "other",
+ KnowledgeBaseID: "kb",
+ Content: "Opening greeting.",
+ IsEnabled: true,
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "a": a,
+ "p": {
+ ID: "p",
+ ChunkType: types.ChunkTypeParentText,
+ },
+ },
+ children: map[string][]*types.Chunk{"p": {
+ a,
+ b,
+ disabled,
+ foreign,
+ }},
+ }
+ p := &PluginMerge{chunkRepo: repo}
+ merged := &types.SearchResult{
+ ID: "a",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ Content: "Opening greeting.\n\nExact later quotation.",
+ ContentRewritten: true,
+ }
+ p.attachCitationSources(context.Background(), []*types.SearchResult{merged})
+ got := expandCitationSources([]*types.SearchResult{merged, merged})
+ require.Len(t, got, 2)
+ require.Equal(t, "a", got[0].ID)
+ require.Equal(t, a.Content, got[0].Content)
+ require.Equal(t, "b", got[1].ID)
+ require.Equal(t, b.Content, got[1].Content)
+ require.Empty(t, got[0].SubChunkID)
+}
+
+func TestExpandedEvidenceFailsClosedAndPreservesPlainSources(t *testing.T) {
+ got := expandCitationSources([]*types.SearchResult{
+ {
+ ID: "expanded",
+ Content: "parent text under child ID",
+ ContentRewritten: true,
+ },
+ {
+ ID: "plain",
+ Content: "actual source",
+ },
+ })
+ require.Len(t, got, 1)
+ require.Equal(t, "plain", got[0].ID)
+}
+
+func TestModelHandlesReferenceTheActualExpandedChild(t *testing.T) {
+ a := &types.SearchResult{
+ ID: "opening",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ KnowledgeTitle: "Report",
+ Content: "Opening greeting.",
+ }
+ b := &types.SearchResult{
+ ID: "later",
+ KnowledgeID: "doc",
+ KnowledgeBaseID: "kb",
+ KnowledgeTitle: "Report",
+ ChunkIndex: 1,
+ Content: "Exact later quotation.",
+ }
+ rendered := "old expanded context"
+ manage := &types.ChatManage{
+ PipelineRequest: types.PipelineRequest{
+ Query: "question",
+ SummaryConfig: types.SummaryConfig{Prompt: "system"},
+ },
+ PipelineState: types.PipelineState{
+ RenderedContexts: rendered,
+ UserContent: rendered,
+ MergeResult: []*types.SearchResult{{
+ ID: "opening",
+ ContentRewritten: true,
+ CitationSources: []*types.SearchResult{
+ a,
+ b,
+ },
+ }},
+ },
+ }
+ messages, registry := prepareMessagesWithModelContext(context.Background(), manage)
+ require.Contains(t, messages[1].Content, ``), `chunk_id="later"`)
+ require.Contains(t, registry.DecodeOutputText(``), `chunk_id="opening"`)
+ manage.MergeResult[0].CitationSources = nil
+ messages, _ = prepareMessagesWithModelContext(context.Background(), manage)
+ require.NotContains(t, messages[1].Content, rendered)
+ require.Contains(t, messages[1].Content, "could not be verified")
+}
diff --git a/internal/application/service/chat_pipeline/merge.go b/internal/application/service/chat_pipeline/merge.go
index 1b370db3..9d131908 100644
--- a/internal/application/service/chat_pipeline/merge.go
+++ b/internal/application/service/chat_pipeline/merge.go
@@ -92,6 +92,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
mergedChunks = p.dedup(ctx, "final_dedup", mergedChunks)
mergedChunks = removePartialOverlaps(ctx, mergedChunks)
+ p.attachCitationSources(ctx, mergedChunks)
chatManage.MergeResult = mergedChunks
return next()
}
diff --git a/internal/application/service/chat_pipeline/references.go b/internal/application/service/chat_pipeline/references.go
index 234149f2..aa69336c 100644
--- a/internal/application/service/chat_pipeline/references.go
+++ b/internal/application/service/chat_pipeline/references.go
@@ -30,7 +30,7 @@ func prepareMessagesWithModelContext(
return messages, registry
}
- ordered := orderedPipelineReferences(chatManage)
+ ordered := expandCitationSources(orderedPipelineReferences(chatManage))
knowledgeResults := make([]*types.SearchResult, 0, len(ordered))
knowledgeRows := make([]map[string]interface{}, 0, len(ordered))
webRows := make([]map[string]interface{}, 0)
@@ -80,7 +80,8 @@ func prepareMessagesWithModelContext(
}
modelContexts := strings.Join(contextParts, "\n")
if strings.TrimSpace(modelContexts) == "" {
- return messages, registry
+ modelContexts = "Retrieved source bodies could not be verified. Do not cite or infer facts from " +
+ "unavailable retrieval evidence."
}
last := len(messages) - 1
diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go
index bfe770d5..6cae3464 100644
--- a/internal/application/service/chunk.go
+++ b/internal/application/service/chunk.go
@@ -568,6 +568,10 @@ func (s *chunkService) UpdateDocumentChunk(
chunk.SourceContent = chunk.Content
}
bodyChanged := newContent != chunk.Content
+ if bodyChanged {
+ // Edited evidence no longer maps to the uploaded original.
+ chunk.SourceLocators = nil
+ }
chunk.Content = newContent
chunk.IsEnabled = newEnabled
chunk.ContentRevision++
@@ -747,6 +751,7 @@ func (s *chunkService) rebuildParentContent(ctx context.Context, edited *types.C
for _, repl := range replacements {
baseRunes = append(append(append([]rune{}, baseRunes[:repl.start]...), []rune(repl.content)...), baseRunes[repl.end:]...)
}
+ parent.SourceLocators = nil // Child edits invalidate the original parent evidence.
parent.Content = string(baseRunes)
for _, conflict := range conflicts {
parent.Content = searchutil.JoinChunkContent(parent.Content, conflict.content, "\n\n")
diff --git a/internal/application/service/chunk_edit_parent_test.go b/internal/application/service/chunk_edit_parent_test.go
index 1d06f7d6..ab0ea2d6 100644
--- a/internal/application/service/chunk_edit_parent_test.go
+++ b/internal/application/service/chunk_edit_parent_test.go
@@ -157,7 +157,8 @@ func TestUpdateDocumentChunkPreservesGeneratedQuestionsAcrossRevision(t *testing
repo := &editableChunkRepo{chunk: &types.Chunk{
ID: "chunk", TenantID: 1, KnowledgeID: "knowledge", KnowledgeBaseID: "kb",
Content: "old body", SourceContent: "old body", ContentRevision: 0,
- ChunkType: types.ChunkTypeText, IsEnabled: true, IndexStatus: "ready", Metadata: metadataJSON,
+ SourceLocators: types.SourceLocators{{Type: "pdf", Page: 1, Mapping: "exact", Quote: "old body"}},
+ ChunkType: types.ChunkTypeText, IsEnabled: true, IndexStatus: "ready", Metadata: metadataJSON,
}}
service := &chunkService{
chunkRepository: repo,
@@ -182,6 +183,9 @@ func TestUpdateDocumentChunkPreservesGeneratedQuestionsAcrossRevision(t *testing
if err != nil {
t.Fatalf("update chunk: %v", err)
}
+ if len(updated.SourceLocators) != 0 {
+ t.Fatal("edited content retained source positions from the original")
+ }
updatedMetadata, err := updated.DocumentMetadata()
if err != nil {
t.Fatal(err)
@@ -219,6 +223,7 @@ func TestRebuildParentContentPreservesConflictingEdits(t *testing.T) {
parent: &types.Chunk{
ID: "parent", TenantID: 1, ChunkType: types.ChunkTypeParentText,
SourceContent: "abcdefghij", Content: "abcdefghij", StartAt: 0, EndAt: 10,
+ SourceLocators: types.SourceLocators{{Type: "pdf", Page: 1}},
},
children: []*types.Chunk{
{
@@ -240,6 +245,9 @@ func TestRebuildParentContentPreservesConflictingEdits(t *testing.T) {
if repo.updated == nil {
t.Fatal("parent was not updated")
}
+ if len(repo.updated.SourceLocators) != 0 {
+ t.Fatal("rebuilt parent retained stale source positions")
+ }
for _, want := range []string{"OLDER EDIT BODY", "NEWER EDIT BODY"} {
if !strings.Contains(repo.updated.Content, want) {
t.Fatalf("rebuilt parent lost %q: %q", want, repo.updated.Content)
diff --git a/internal/application/service/docreader_failure.go b/internal/application/service/docreader_failure.go
new file mode 100644
index 00000000..f9e926fa
--- /dev/null
+++ b/internal/application/service/docreader_failure.go
@@ -0,0 +1,30 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "net"
+ "strings"
+
+ werrors "github.com/Tencent/WeKnora/internal/errors"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+// Keep actionable public messages separate from raw transport diagnostics,
+// which may contain internal addresses or credentials and belong in error_detail.
+func docReaderFailure(err error) (code, message string) {
+ if errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded ||
+ strings.Contains(err.Error(), "docreader call timeout") {
+ return werrors.ErrCodeDocReaderTimeout,
+ "Document parsing service timed out. Check service health and load before retrying."
+ }
+ var networkError *net.OpError
+ if status.Code(err) == codes.Unavailable || errors.As(err, &networkError) {
+ return werrors.ErrCodeDocReaderUnavailable,
+ "Cannot connect to the document parsing service, or the connection was interrupted. " +
+ "Check that DocReader is running and healthy, then retry."
+ }
+ return werrors.ErrCodeDocReaderParseFailed,
+ "Document parsing failed. Check the file format and the DocReader logs for this attempt."
+}
diff --git a/internal/application/service/docreader_failure_test.go b/internal/application/service/docreader_failure_test.go
new file mode 100644
index 00000000..544c937a
--- /dev/null
+++ b/internal/application/service/docreader_failure_test.go
@@ -0,0 +1,52 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "strings"
+ "testing"
+
+ werrors "github.com/Tencent/WeKnora/internal/errors"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+func TestDocReaderFailure(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ code string
+ }{
+ {
+ "scanned PDF wrapped connection failure",
+ fmt.Errorf("anydoc fallback: %w",
+ fmt.Errorf("read stream: %w",
+ status.Error(codes.Unavailable,
+ "private-address secret-token"))),
+ werrors.ErrCodeDocReaderUnavailable,
+ },
+ {
+ "HTTP connection failure",
+ &net.OpError{
+ Op: "dial",
+ Err: errors.New("private-address secret-token"),
+ },
+ werrors.ErrCodeDocReaderUnavailable,
+ },
+ {"context timeout", fmt.Errorf("parse: %w", context.DeadlineExceeded), werrors.ErrCodeDocReaderTimeout},
+ {"RPC timeout", status.Error(codes.DeadlineExceeded, "private-address"), werrors.ErrCodeDocReaderTimeout},
+ {"parser failure", errors.New("private-address secret-token"), werrors.ErrCodeDocReaderParseFailed},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ code, message := docReaderFailure(tc.err)
+ if code != tc.code || message == "" {
+ t.Fatalf("got %q %q, want code %q", code, message, tc.code)
+ }
+ if strings.Contains(message, "private-address") || strings.Contains(message, "secret-token") {
+ t.Fatal("raw transport diagnostics leaked into public message")
+ }
+ })
+ }
+}
diff --git a/internal/application/service/image_chunk_owner.go b/internal/application/service/image_chunk_owner.go
new file mode 100644
index 00000000..00c3aeb4
--- /dev/null
+++ b/internal/application/service/image_chunk_owner.go
@@ -0,0 +1,24 @@
+package service
+
+import (
+ "github.com/Tencent/WeKnora/internal/searchutil"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// A missing association is preferable to attaching evidence to an unrelated
+// first paragraph. Exact Markdown targets also avoid URL-prefix collisions.
+func imageChunkOwner(url string, chunks []types.ParsedChunk) string {
+ if url == "" {
+ return ""
+ }
+ owner := ""
+ for _, c := range chunks {
+ if c.ChunkID != "" && searchutil.ImageURLsInContent(c.Content)[url] {
+ if owner != "" && owner != c.ChunkID {
+ return ""
+ }
+ owner = c.ChunkID
+ }
+ }
+ return owner
+}
diff --git a/internal/application/service/image_chunk_owner_test.go b/internal/application/service/image_chunk_owner_test.go
new file mode 100644
index 00000000..33110339
--- /dev/null
+++ b/internal/application/service/image_chunk_owner_test.go
@@ -0,0 +1,41 @@
+package service
+
+import (
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestImageChunkOwnerUsesExactTargets(t *testing.T) {
+ chunks := []types.ParsedChunk{
+ {
+ ChunkID: "first",
+ Content: "unrelated",
+ },
+ {
+ ChunkID: "prefix",
+ Content: "",
+ },
+ {
+ ChunkID: "actual",
+ Content: "",
+ },
+ }
+ require.Equal(t, "actual", imageChunkOwner("resource://abc", chunks))
+ require.Empty(t, imageChunkOwner("resource://missing", chunks))
+ require.Empty(t, imageChunkOwner("", chunks))
+ require.Empty(t,
+ imageChunkOwner("resource://abc",
+ append(chunks,
+ types.ParsedChunk{
+ ChunkID: "duplicate",
+ Content: "",
+ })))
+ require.Empty(t,
+ imageChunkOwner("resource://abc",
+ []types.ParsedChunk{{
+ ChunkID: "prose",
+ Content: "resource://abc",
+ }}))
+}
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index 6994c9ea..b0e5c7db 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -2,6 +2,7 @@ package service
import (
"context"
+ "crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -317,11 +318,13 @@ func (s *ImageMultimodalService) processImage(
logger.Errorf(ctx, "[ImageMultimodal] Skip unreadable image %s: %v", payload.ImageURL, readErr)
out["skipped"] = "unreadable_image"
out["read_error"] = readErr.Error()
- return nil
+ handleErr = fmt.Errorf("read source image: %w", readErr)
+ return handleErr
}
out["image_bytes"] = len(imgBytes)
imageInfo := types.ImageInfo{
+ SHA256: fmt.Sprintf("%x", sha256.Sum256(imgBytes)),
URL: payload.ImageURL,
OriginalURL: payload.ImageURL,
}
diff --git a/internal/application/service/image_multimodal_attrs_test.go b/internal/application/service/image_multimodal_attrs_test.go
index 36f78e85..4029a865 100644
--- a/internal/application/service/image_multimodal_attrs_test.go
+++ b/internal/application/service/image_multimodal_attrs_test.go
@@ -3,7 +3,9 @@ package service
import (
"bytes"
"context"
+ "crypto/sha256"
"encoding/json"
+ "errors"
"fmt"
"io"
"strings"
@@ -333,6 +335,15 @@ func TestProcessImageObservesAndDescribes(t *testing.T) {
if len(repo.created) != 2 {
t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created))
}
+ for _, chunk := range repo.created {
+ var images []types.ImageInfo
+ if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 {
+ t.Fatalf("invalid image info: %v", err)
+ }
+ if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) {
+ t.Fatal("stored image fingerprint does not identify the bytes sent to OCR")
+ }
+ }
if got := persistedAttrCounts(t, repo.created, "contain.text"); !equalCounts(got, map[string]int{"block": 2}) {
t.Errorf("persisted contain.text = %v, want block on both chunks", got)
}
@@ -553,6 +564,15 @@ func TestProcessImageProseAnswerIsStillCaptionedAndOCRed(t *testing.T) {
if len(repo.created) != 2 {
t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created))
}
+ for _, chunk := range repo.created {
+ var images []types.ImageInfo
+ if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 {
+ t.Fatalf("invalid image info: %v", err)
+ }
+ if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) {
+ t.Fatal("stored image fingerprint does not identify the bytes sent to OCR")
+ }
+ }
}
// TestProcessImageLabelOnlyAnswerKeepsAttrsWithoutCaption covers the one reply
@@ -654,11 +674,19 @@ func TestProcessImageCaptionOCRPipeline(t *testing.T) {
if len(repo.created) != 2 {
t.Fatalf("persisted chunks = %d, want OCR + caption", len(repo.created))
}
+ for _, chunk := range repo.created {
+ var images []types.ImageInfo
+ if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err != nil || len(images) != 1 {
+ t.Fatalf("invalid image info: %v", err)
+ }
+ if images[0].SHA256 != fmt.Sprintf("%x", sha256.Sum256(fileSvc.body)) {
+ t.Fatal("stored image fingerprint does not identify the bytes sent to OCR")
+ }
+ }
}
-// TestProcessImageSkipsUnreadableImage checks that an unreadable object is
-// reported and counted rather than failing the task.
-func TestProcessImageSkipsUnreadableImage(t *testing.T) {
+// Unreadable storage must return its cause so Handle retries before finalizing.
+func TestProcessImageReturnsUnreadableImageError(t *testing.T) {
t.Parallel()
fileSvc := &attrsFileService{err: fmt.Errorf("object is gone")}
@@ -678,7 +706,12 @@ func TestProcessImageSkipsUnreadableImage(t *testing.T) {
ImageActions: types.DefaultImageActions(),
}
- out := runProcessImage(t, svc, payload, fake)
+ out := types.JSONMap{}
+ err := svc.processImage(context.Background(), payload, fake, types.VLMConfig{}, noopSpanTracker{}, out)
+ if !errors.Is(err, fileSvc.err) {
+ t.Fatalf("storage error was swallowed: %v", err)
+ }
+
if got := out["skipped"]; got != "unreadable_image" {
t.Errorf("skipped = %v, want unreadable_image", got)
}
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index ad4cfce6..07e49da3 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -3872,6 +3872,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
if convertResult != nil {
parsedMarkdown = convertResult.MarkdownContent
sourceBlocks = convertResult.SourceBlocks
+ for i := range sourceBlocks {
+ sourceBlocks[i].Locator.SourceHash = knowledge.FileHash
+ }
}
sanitizeReadResult(convertResult)
if convertResult != nil {
@@ -4109,16 +4112,11 @@ func (s *knowledgeService) convert(
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
if err != nil {
- // Distinguish DocReader timeout (a knowable user-facing
- // failure) from generic read errors so the UI can suggest
- // "split this large file" specifically when relevant.
- code := werrors.ErrCodeDocReaderParseFailed
- if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), "docreader call timeout") {
- code = werrors.ErrCodeDocReaderTimeout
- }
+ code, message := docReaderFailure(err)
+ logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)
s.failStage(ctx, knowledge.ID, types.StageDocReader,
- code, "document read failed", err)
- return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
+ code, message, err)
+ return s.failKnowledge(ctx, knowledge, isLastRetry, "%s", message)
}
sanitizeReadResult(result)
if result.Error != "" {
@@ -4292,15 +4290,9 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
for idx, img := range images {
// Match image to the ParsedChunk whose content contains the image URL.
// ChunkID was populated by processChunks with the real DB UUID.
- chunkID := ""
- for _, c := range chunks {
- if strings.Contains(c.Content, img.ServingURL) {
- chunkID = c.ChunkID
- break
- }
- }
- if chunkID == "" && len(chunks) > 0 {
- chunkID = chunks[0].ChunkID
+ chunkID := imageChunkOwner(img.ServingURL, chunks)
+ if chunkID == "" {
+ logger.Warnf(ctx, "Image has no owning text chunk: knowledge=%s image=%s", knowledge.ID, img.ServingURL)
}
payload := types.ImageMultimodalPayload{
diff --git a/internal/infrastructure/docparser/grpc_parser.go b/internal/infrastructure/docparser/grpc_parser.go
index ac6e6ccd..e78b2ca4 100644
--- a/internal/infrastructure/docparser/grpc_parser.go
+++ b/internal/infrastructure/docparser/grpc_parser.go
@@ -105,7 +105,7 @@ func (p *GRPCDocumentReader) Close() error {
return nil
}
-var errNotConnected = fmt.Errorf("docreader service not connected")
+var errNotConnected = status.Error(codes.Unavailable, "docreader service not connected")
func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
p.mu.RLock()
diff --git a/internal/infrastructure/docparser/mineru_layout.go b/internal/infrastructure/docparser/mineru_layout.go
index a461e579..169f21eb 100644
--- a/internal/infrastructure/docparser/mineru_layout.go
+++ b/internal/infrastructure/docparser/mineru_layout.go
@@ -4,6 +4,9 @@ import (
"archive/zip"
"bytes"
"encoding/json"
+ "fmt"
+ "html"
+ "math"
"regexp"
"sort"
"strings"
@@ -20,7 +23,7 @@ const minerUBBoxScale = 1000.0
type minerUContentItem struct {
Type string `json:"type"`
Text string `json:"text"`
- PageIdx int `json:"page_idx"`
+ PageIdx *int `json:"page_idx"`
BBox []float64 `json:"bbox"`
TableBody string `json:"table_body"`
TableCaption json.RawMessage `json:"table_caption"`
@@ -82,8 +85,8 @@ func decodeMinerUContentList(raw []byte) []minerUContentItem {
return items
}
-// minerUContentListFromZip returns the shallowest *content_list.json of a
-// MinerU result package, or nil.
+// minerUContentListFromZip prefers supported MiddleJson 2.0 geometry and
+// falls back to legacy content_list.json for older MinerU packages.
func minerUContentListFromZip(zipData []byte) []byte {
zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
if err != nil {
@@ -92,7 +95,7 @@ func minerUContentListFromZip(zipData []byte) []byte {
var candidates []*zip.File
for _, f := range zr.File {
name := strings.ToLower(f.Name)
- if strings.HasSuffix(name, "content_list.json") && !strings.Contains(name, "content_list_v2") {
+ if strings.HasSuffix(name, "middle_json.json") || strings.HasSuffix(name, "content_list.json") {
candidates = append(candidates, f)
}
}
@@ -100,13 +103,34 @@ func minerUContentListFromZip(zipData []byte) []byte {
return nil
}
sort.Slice(candidates, func(i, j int) bool {
- return strings.Count(candidates[i].Name, "/") < strings.Count(candidates[j].Name, "/")
+ mi,
+ mj := strings.HasSuffix(strings.ToLower(candidates[i].Name),
+ "middle_json.json"),
+ strings.HasSuffix(strings.ToLower(candidates[j].Name),
+ "middle_json.json")
+ if mi != mj {
+ return mi
+ }
+ di, dj := strings.Count(candidates[i].Name, "/"), strings.Count(candidates[j].Name, "/")
+ if di != dj {
+ return di < dj
+ }
+ return candidates[i].Name < candidates[j].Name
})
- data, err := readZipEntryBytes(candidates[0])
- if err != nil {
- return nil
+ for _, candidate := range candidates {
+ data, err := readZipEntryBytes(candidate)
+ if err != nil {
+ continue
+ }
+ if strings.HasSuffix(strings.ToLower(candidate.Name), "middle_json.json") {
+ var doc minerUMiddle
+ if json.Unmarshal(data, &doc) != nil || doc.Schema != "docvortex.middle" || doc.Version != "2.0" {
+ continue
+ }
+ }
+ return data
}
- return data
+ return nil
}
// minerUSourceBlocks aligns MinerU's layout blocks against its markdown so
@@ -119,13 +143,26 @@ func minerUSourceBlocks(markdown string, contentList []byte, fileType string) []
if ft != "pdf" && !IsImageFormat(ft) {
return nil
}
+ raw := bytes.TrimSpace(contentList)
+ if len(raw) > 0 && raw[0] == '{' {
+ return minerUMiddleSourceBlocks(markdown, raw)
+ }
items := decodeMinerUContentList(contentList)
if len(items) == 0 {
return nil
}
units := make([]sourceloc.Unit, 0, len(items))
- for _, it := range items {
- loc := types.SourceLocator{Type: types.SourceLocatorPDF, Page: it.PageIdx + 1}
+ for i, it := range items {
+ if it.PageIdx == nil || *it.PageIdx < 0 {
+ continue
+ }
+ loc := types.SourceLocator{
+ Type: types.SourceLocatorPDF,
+ Page: *it.PageIdx + 1,
+ SourceID: fmt.Sprintf("mineru:legacy:%d:%d",
+ *it.PageIdx,
+ i),
+ }
if bbox, ok := minerUBBox(it.BBox); ok {
loc.BBox = bbox
}
@@ -142,7 +179,7 @@ func minerUBBox(b []float64) ([]float64, bool) {
return nil, false
}
for _, v := range b {
- if v < 0 || v > minerUBBoxScale {
+ if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > minerUBBoxScale {
return nil, false
}
}
@@ -152,3 +189,110 @@ func minerUBBox(b []float64) ([]float64, bool) {
}
return out, true
}
+
+// MiddleJson 2.0 uses [0,1] normalized boxes (NOT legacy 0..1000).
+// Child annotations keep their own geometry. Inline spans have no geometry.
+type minerUMiddleBlock struct {
+ Type string `json:"type"`
+ Index *int `json:"index"`
+ BBox []float64 `json:"bbox"`
+ Content json.RawMessage `json:"content"`
+}
+
+type minerUMiddle struct {
+ Schema string `json:"schema"`
+ Version string `json:"schema_version"`
+ Pages []struct {
+ PageIdx *int `json:"page_idx"`
+ Blocks []minerUMiddleBlock `json:"blocks"`
+ } `json:"pages"`
+}
+
+func minerUMiddleSourceBlocks(markdown string, raw []byte) []types.SourceBlock {
+ var doc minerUMiddle
+ if json.Unmarshal(raw, &doc) != nil || doc.Schema != "docvortex.middle" || doc.Version != "2.0" {
+ return nil
+ }
+ var units []sourceloc.Unit
+ previousPage := -1
+ for _, page := range doc.Pages {
+ if page.PageIdx == nil || *page.PageIdx <= previousPage {
+ return nil
+ }
+ previousPage = *page.PageIdx
+ previousBlock := -1
+ for _, block := range page.Blocks {
+ if block.Index == nil || *block.Index <= previousBlock {
+ return nil
+ }
+ previousBlock = *block.Index
+ ref := fmt.Sprintf("mineru:2:%d:%d", *page.PageIdx, *block.Index)
+ units = append(units, minerUMiddleUnits(block, *page.PageIdx+1, ref, nil)...)
+ }
+ }
+ return sourceloc.Align(markdown, units)
+}
+
+func minerUMiddleUnits(block minerUMiddleBlock, page int, ref string, parentBox []float64) []sourceloc.Unit {
+ box := block.BBox
+ if len(box) == 0 {
+ box = parentBox
+ }
+ loc := types.SourceLocator{Type: types.SourceLocatorPDF, Page: page, SourceID: ref}
+ if normalizedMinerUBox(box) {
+ loc.BBox = append([]float64(nil), box...)
+ }
+ // These containers contain blocks; text and hyperlinks contain inline spans.
+ switch block.Type {
+ case "list", "index", "table", "image", "chart", "code":
+ var children []minerUMiddleBlock
+ if json.Unmarshal(block.Content, &children) != nil {
+ return nil
+ }
+ var out []sourceloc.Unit
+ for i, child := range children {
+ out = append(out, minerUMiddleUnits(child, page, fmt.Sprintf("%s/%d", ref, i), box)...)
+ }
+ return out
+ default:
+ text := minerUMiddleText(block.Content)
+ if block.Type == "table_body" || block.Type == "chart_body" {
+ text = html.UnescapeString(minerUHTMLTagRe.ReplaceAllString(text, " "))
+ }
+ if strings.TrimSpace(text) == "" {
+ return nil
+ }
+ return []sourceloc.Unit{{Text: text, Locator: loc}}
+ }
+}
+
+func minerUMiddleText(raw json.RawMessage) string {
+ var text string
+ if json.Unmarshal(raw, &text) == nil {
+ return text
+ }
+ var spans []struct {
+ Type string `json:"type"`
+ Content json.RawMessage `json:"content"`
+ }
+ if json.Unmarshal(raw, &spans) != nil {
+ return ""
+ }
+ var out strings.Builder
+ for _, span := range spans {
+ out.WriteString(minerUMiddleText(span.Content))
+ }
+ return out.String()
+}
+
+func normalizedMinerUBox(box []float64) bool {
+ if len(box) != 4 || box[0] >= box[2] || box[1] >= box[3] {
+ return false
+ }
+ for _, v := range box {
+ if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > 1 {
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/infrastructure/docparser/mineru_layout_precision_test.go b/internal/infrastructure/docparser/mineru_layout_precision_test.go
new file mode 100644
index 00000000..faea0646
--- /dev/null
+++ b/internal/infrastructure/docparser/mineru_layout_precision_test.go
@@ -0,0 +1,172 @@
+package docparser
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "math"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// Contract fixture for DocVortex MiddleJson 2.0. Bboxes are normalized
+// fractions, page indices are original source indices, and captions are
+// child blocks. This is deliberately not a legacy content_list payload.
+const middleFixture = `{
+ "schema": "docvortex.middle",
+ "schema_version": "2.0",
+ "is_full_document": true,
+ "metadata": {
+ "file_suffix": "pdf",
+ "producer": {
+ "name": "mineru",
+ "version": "4.0.0"
+ }
+ },
+ "pages": [
+ {
+ "page_idx": 4,
+ "blocks": [
+ {
+ "type": "text",
+ "index": 0,
+ "bbox": [
+ 0.1,
+ 0.2,
+ 0.9,
+ 0.3
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "设备"
+ },
+ {
+ "type": "hyperlink",
+ "url": "https://example.org",
+ "content": [
+ {
+ "type": "text",
+ "content": "验收合格"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "type": "table",
+ "index": 1,
+ "bbox": [
+ 0.1,
+ 0.4,
+ 0.9,
+ 0.8
+ ],
+ "content": [
+ {
+ "type": "table_caption",
+ "bbox": [
+ 0.1,
+ 0.4,
+ 0.9,
+ 0.45
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "设备参数表"
+ }
+ ]
+ },
+ {
+ "type": "table_body",
+ "bbox": [
+ 0.1,
+ 0.46,
+ 0.9,
+ 0.8
+ ],
+ "content": ""
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}`
+
+func TestMinerUMiddleNormalizedGeometryAndNestedBlocks(t *testing.T) {
+ blocks := minerUSourceBlocks("设备验收合格\n\n设备参数表\n\n|限值|1.5毫米|", []byte(middleFixture), "pdf")
+ require.Len(t, blocks, 3)
+ require.Equal(t, 5, blocks[0].Locator.Page)
+ require.Equal(t, []float64{0.1, 0.2, 0.9, 0.3}, blocks[0].Locator.BBox)
+ require.Equal(t, "mineru:2:4:0", blocks[0].Locator.SourceID)
+ require.Equal(t, "exact", blocks[0].Locator.Mapping)
+ require.Equal(t, []float64{0.1, 0.46, 0.9, 0.8}, blocks[2].Locator.BBox)
+}
+
+func TestMinerUMiddleZIPPreferredToLegacy(t *testing.T) {
+ var buf bytes.Buffer
+ z := zip.NewWriter(&buf)
+ for name, body := range map[string]string{
+ "markdown.md": "设备验收合格",
+ "middle_json.json": middleFixture,
+ "doc_content_list.json": "[]",
+ } {
+ w, err := z.Create(name)
+ require.NoError(t, err)
+ _, err = w.Write([]byte(body))
+ require.NoError(t, err)
+ }
+ require.NoError(t, z.Close())
+ require.JSONEq(t, middleFixture, string(minerUContentListFromZip(buf.Bytes())))
+}
+
+func TestMinerUMiddleRejectsUnknownSchemaAndInvalidIndices(t *testing.T) {
+ for _, mutate := range []func(map[string]any){
+ func(d map[string]any) { d["schema"] = "docvortex.model" },
+ func(d map[string]any) { d["schema_version"] = "9.0" },
+ func(d map[string]any) { delete(d["pages"].([]any)[0].(map[string]any), "page_idx") },
+ func(d map[string]any) { d["pages"].([]any)[0].(map[string]any)["page_idx"] = -1 },
+ } {
+ var d map[string]any
+ require.NoError(t, json.Unmarshal([]byte(middleFixture), &d))
+ mutate(d)
+ raw, err := json.Marshal(d)
+ require.NoError(t, err)
+ require.Empty(t, minerUSourceBlocks("设备验收合格", raw, "pdf"))
+ }
+}
+
+func TestMinerULegacyMissingPageIsNotPageOne(t *testing.T) {
+ require.Empty(t, minerUSourceBlocks("设备验收合格", []byte(`[{"type":"text","text":"设备验收合格"}]`), "pdf"))
+}
+
+func TestMinerUBoxesRejectNonFiniteAndWrongUnits(t *testing.T) {
+ for _, box := range [][]float64{{0, 0, math.NaN(), 1}, {0, 0, math.Inf(1), 1}, {0, 0, 900, 600}, {.9, 0, .1, 1}} {
+ require.False(t, normalizedMinerUBox(box))
+ }
+ _, ok := minerUBBox([]float64{0, 0, math.NaN(), 100})
+ require.False(t, ok)
+}
+
+func TestMinerULegacyZIPWithUnsupportedMiddleKeepsContentList(t *testing.T) {
+ var buf bytes.Buffer
+ z := zip.NewWriter(&buf)
+ legacy := `[{"type":"text","text":"设备验收合格","page_idx":1,"bbox":[100,200,900,300]}]`
+ for name, body := range map[string]string{
+ "doc_middle_json.json": `{"pdf_info":[]}`,
+ "doc_content_list.json": legacy,
+ } {
+ w, err := z.Create(name)
+ require.NoError(t, err)
+ _, err = w.Write([]byte(body))
+ require.NoError(t, err)
+ }
+ require.NoError(t, z.Close())
+ blocks := minerUSourceBlocks("设备验收合格", minerUContentListFromZip(buf.Bytes()), "pdf")
+ require.Len(t, blocks, 1)
+ require.Equal(t, 2, blocks[0].Locator.Page)
+ require.Equal(t, []float64{.1, .2, .9, .3}, blocks[0].Locator.BBox)
+}
diff --git a/internal/modelcontext/citations.go b/internal/modelcontext/citations.go
index 98e69098..28bdfb71 100644
--- a/internal/modelcontext/citations.go
+++ b/internal/modelcontext/citations.go
@@ -22,6 +22,11 @@ const citationEnabledProtocolPrompt = `
- Source citations are enabled for this answer. Cite a knowledge chunk with exactly and a web page with exactly .
- Cite only cN/wN handles backed by tool results for the current task, and only when that source supports the adjacent
claim. Never cite dN/bN.
+- Each cN covers only its own supplied text or image evidence, not every chunk of the same document.
+ For a direct quotation, verify that the quoted words occur in the cited cN. Use the later chunk's handle
+ when the words occur there; never reuse a document's first handle for unrelated passages.
+- OCR watermarks, timestamps and image captions are observations of an image. Do not turn a watermark slogan
+ into a verified service commitment or a caption into a factual document statement without supporting text.
- Handles in historical answers, tool arguments, or the bound knowledge-base directory are for navigation, not current
evidence. Retrieve the relevant source before citing it.
- MCP results are external sources. Use the wN handle for the matching URL in the system-provided
diff --git a/internal/sourceloc/align.go b/internal/sourceloc/align.go
index e95c7395..f8e09f8c 100644
--- a/internal/sourceloc/align.go
+++ b/internal/sourceloc/align.go
@@ -1,6 +1,7 @@
package sourceloc
import (
+ "html"
"sort"
"strings"
"unicode"
@@ -16,69 +17,95 @@ type Unit struct {
Locator types.SourceLocator
}
-const (
- // alignKeyRunes is how much of a unit's normalized text is searched for.
- alignKeyRunes = 32
- // alignMinRunes skips units too short to place reliably.
- alignMinRunes = 2
- // alignWindowRunes bounds how far past the cursor a unit is searched, so
- // a unit the parser dropped cannot drag the cursor across the document.
- alignWindowRunes = 20000
-)
+// Skip units too short to place reliably.
+const alignMinRunes = 2
-// Align places units, which are in document order, into markdown and returns
-// one block per placed unit. A block runs from where its unit was found to
-// where the next placed unit starts, so the blocks tile the markdown from the
-// first placed unit on. Units are matched on letters and digits only, which
-// makes the match indifferent to Markdown syntax, whitespace and
-// punctuation the parser added or dropped.
+// Align maps complete source units, never extending a hit through unmatched
+// content. Unknown gaps must stay unknown: assigning them to the preceding
+// block can point a citation at an entirely different page.
func Align(markdown string, units []Unit) []types.SourceBlock {
norm := normalizeMarkdown(markdown)
- if norm.text == "" || len(units) == 0 {
- return nil
+ runes := []rune(markdown)
+ var blocks []types.SourceBlock
+ cursor, floor := 0, 0
+ counts := make(map[string]int)
+ matches := make(map[string][]int)
+ for _, u := range units {
+ counts[normalizeKey(u.Text)]++
}
- type hit struct {
- pos int // rune offset in markdown
- unit int
- }
- var hits []hit
- cursor := 0 // byte offset into norm.text
- for i, u := range units {
+ for _, u := range units {
key := normalizeKey(u.Text)
if utf8.RuneCountInString(key) < alignMinRunes {
continue
}
- at, width := norm.find(key, cursor)
- if at < 0 {
+ // Repeated source text is safe only when every occurrence is accounted
+ // for by a source unit in reading order (not a TOC or an omitted block).
+ positions, known := matches[key]
+ if !known {
+ positions = normalizedMatches(norm.text, key)
+ matches[key] = positions
+ }
+ if len(positions) != counts[key] {
continue
}
- hits = append(hits, hit{pos: norm.pos[at], unit: i})
- cursor = at + width
- }
- if len(hits) == 0 {
- return nil
- }
- runes := []rune(markdown)
- prev := 0
- for i := range hits {
- hits[i].pos = lineLeadStart(runes, hits[i].pos, prev)
- prev = hits[i].pos
- }
- total := len(runes)
- blocks := make([]types.SourceBlock, 0, len(hits))
- for i, h := range hits {
- end := total
- if i+1 < len(hits) {
- end = hits[i+1].pos
- }
- if end <= h.pos {
+ matchIndex := sort.SearchInts(positions, cursor)
+ if matchIndex == len(positions) {
continue
}
- blocks = append(blocks, types.SourceBlock{Start: h.pos, End: end, Locator: units[h.unit].Locator})
+ at := positions[matchIndex]
+ endByte := at + len(key)
+ start := lineLeadStart(runes, norm.pos[at], floor)
+ end := norm.end[endByte-1]
+ // Include trailing markup/URLs on the matched line and blank lines,
+ // but stop before the next visible text's line. This also preserves
+ // an image's destination without swallowing another paragraph.
+ if endByte == len(norm.text) {
+ end = len(runes)
+ } else {
+ next := norm.pos[endByte]
+ lead := next
+ for lead > 0 && runes[lead-1] != '\n' {
+ lead--
+ }
+ if lead >= end {
+ end = lead
+ }
+ }
+ loc := u.Locator
+ loc.Mapping = "exact"
+ blocks = append(blocks, types.SourceBlock{Start: start, End: end, Locator: loc})
+ cursor, floor = endByte, end
}
return blocks
}
+// normalizedMatches rejects a number embedded in a different numeric value.
+func normalizedMatches(text, key string) []int {
+ var out []int
+ if key == "" {
+ return out
+ }
+ first, _ := utf8.DecodeRuneInString(key)
+ last, _ := utf8.DecodeLastRuneInString(key)
+ for from := 0; from <= len(text)-len(key); {
+ i := strings.Index(text[from:], key)
+ if i < 0 {
+ break
+ }
+ at := from + i
+ before, _ := utf8.DecodeLastRuneInString(text[:at])
+ after, _ := utf8.DecodeRuneInString(text[at+len(key):])
+ left := unicode.IsDigit(first) && (unicode.IsDigit(before) || strings.ContainsRune(".,:/+−-", before))
+ right := unicode.IsDigit(last) && (unicode.IsDigit(after) || strings.ContainsRune(".,:/%‰", after))
+ if !left && !right {
+ out = append(out, at)
+ }
+ _, size := utf8.DecodeRuneInString(text[at:])
+ from = at + size
+ }
+ return out
+}
+
// lineLeadStart moves pos back to the start of its line when everything
// before it on the line is markup or numbering ("## ", "| ", "1. ", "(2) "),
// which belongs to the unit but was not part of its text. It never crosses
@@ -95,45 +122,13 @@ func lineLeadStart(runes []rune, pos, floor int) int {
return start
}
-// normalized is the letters-and-digits projection of a text: text holds the
+// normalized retains letters, digits and numeric symbols: text holds the
// kept runes, pos[b] the rune offset in the source of the rune starting at
// byte b of text.
type normalized struct {
text string
pos []int
-}
-
-// find searches key (already normalized) at or after byte offset from,
-// within the search window. It tries the key's head first and falls back to
-// a slice from its middle, so a unit whose opening differs (a list number
-// the parser rendered, a leading image) still lands. It returns the byte
-// offset of the match and how far the cursor should advance.
-func (n normalized) find(key string, from int) (int, int) {
- limit := len(n.text)
- if w := from + alignWindowRunes*3; w < limit {
- limit = w
- }
- window := n.text[from:limit]
- head := firstRunes(key, alignKeyRunes)
- if idx := strings.Index(window, head); idx >= 0 {
- return from + idx, len(head)
- }
- runes := []rune(key)
- if len(runes) < alignKeyRunes*2 {
- return -1, 0
- }
- mid := string(runes[len(runes)/2 : len(runes)/2+alignKeyRunes])
- if idx := strings.Index(window, mid); idx >= 0 {
- // Aim at where the unit should begin, not at its middle.
- back := len(string(runes[:len(runes)/2]))
- start := from + idx - back
- start = max(start, from)
- for start < from+idx && !utf8.RuneStart(n.text[start]) {
- start++
- }
- return start, idx + len(mid) - (start - from)
- }
- return -1, 0
+ end []int
}
// normalizeMarkdown projects markdown onto its letters and digits, skipping
@@ -143,6 +138,15 @@ func normalizeMarkdown(markdown string) normalized {
var b strings.Builder
b.Grow(len(markdown))
pos := make([]int, 0, len(markdown))
+ ends := make([]int, 0, len(markdown))
+ emit := func(r rune, start, end int) {
+ before := b.Len()
+ b.WriteRune(r)
+ for k := before; k < b.Len(); k++ {
+ pos = append(pos, start)
+ ends = append(ends, end)
+ }
+ }
runes := []rune(markdown)
for i := 0; i < len(runes); i++ {
r := runes[i]
@@ -158,29 +162,77 @@ func normalizeMarkdown(markdown string) normalized {
continue
}
}
- if nr, ok := foldRune(r); ok {
- start := b.Len()
- b.WriteRune(nr)
- for k := start; k < b.Len(); k++ {
- pos = append(pos, i)
+ if r == '&' {
+ if end := indexRune(runes, ';', i+1, 32); end > 0 {
+ raw := string(runes[i : end+1])
+ if decoded := html.UnescapeString(raw); decoded != raw {
+ context := []rune{' '}
+ if i > 0 {
+ context[0] = runes[i-1]
+ }
+ context = append(context, []rune(decoded)...)
+ context = append(context, ' ')
+ if end+1 < len(runes) {
+ context[len(context)-1] = runes[end+1]
+ }
+ for j := 1; j+1 < len(context); j++ {
+ if nr, ok := matchRune(context, j); ok {
+ emit(nr, i, end+1)
+ }
+ }
+ i = end
+ continue
+ }
}
}
+ if nr, ok := matchRune(runes, i); ok {
+ emit(nr, i, i+1)
+ }
}
- return normalized{text: b.String(), pos: pos}
+ return normalized{text: b.String(), pos: pos, end: ends}
}
// normalizeKey projects plain text onto its letters and digits.
func normalizeKey(text string) string {
var b strings.Builder
- b.Grow(len(text))
- for _, r := range text {
- if nr, ok := foldRune(r); ok {
- b.WriteRune(nr)
+ runes := []rune(text)
+ for i := range runes {
+ if r, ok := matchRune(runes, i); ok {
+ b.WriteRune(r)
}
}
return b.String()
}
+// Preserve distinctions such as 1.5/15, -5/5, 5%/5 and <5/>5.
+func matchRune(rs []rune, i int) (rune, bool) {
+ r := rs[i]
+ if r >= 0xFF01 && r <= 0xFF5E {
+ r -= 0xFEE0
+ }
+ if !strings.ContainsRune(".,:/+−-%‰<>=≤≥≠", r) {
+ return foldRune(r)
+ }
+ before, after := i-1, i+1
+ for before >= 0 && (rs[before] == ' ' || rs[before] == '\t') {
+ before--
+ }
+ for after < len(rs) && (rs[after] == ' ' || rs[after] == '\t') {
+ after++
+ }
+ prevDigit := before >= 0 && unicode.IsDigit(rs[before])
+ nextDigit := after < len(rs) && unicode.IsDigit(rs[after])
+ if ((r == '.' || r == ',' || r == '/' || r == ':') && prevDigit && nextDigit) ||
+ ((r == '-' || r == '+' || r == '−') && nextDigit) ||
+ ((r == '%' || r == '‰') && prevDigit) || strings.ContainsRune("<>=≤≥≠", r) {
+ if r == '−' {
+ r = '-'
+ }
+ return r, true
+ }
+ return foldRune(r)
+}
+
// foldRune keeps letters and digits, lower-cased and with full-width ASCII
// folded to its half-width form.
func foldRune(r rune) (rune, bool) {
@@ -212,17 +264,6 @@ func indexRune(runes []rune, r rune, from, limit int) int {
return -1
}
-func firstRunes(s string, n int) string {
- i := 0
- for idx := range s {
- if i == n {
- return s[:idx]
- }
- i++
- }
- return s
-}
-
// SortBlocks orders blocks by start offset.
func SortBlocks(blocks []types.SourceBlock) {
sort.SliceStable(blocks, func(i, j int) bool { return blocks[i].Start < blocks[j].Start })
diff --git a/internal/sourceloc/assign.go b/internal/sourceloc/assign.go
index 4dba5efe..f81fc430 100644
--- a/internal/sourceloc/assign.go
+++ b/internal/sourceloc/assign.go
@@ -1,6 +1,7 @@
package sourceloc
import (
+ "html"
"regexp"
"sort"
"strings"
@@ -9,10 +10,6 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
-// MaxLocatorsPerChunk caps how many locators one chunk (or merged search
-// result) carries.
-const MaxLocatorsPerChunk = 64
-
// Index answers "which source positions does this range of the final
// Markdown cover" for the chunker's output.
type Index struct {
@@ -51,13 +48,18 @@ func (x *Index) Locators(start, end int) types.SourceLocators {
// Blocks before i all end at or before start.
i := sort.Search(len(x.blocks), func(k int) bool { return x.maxEnd[k] > start })
var out types.SourceLocators
+ covered, partial := start, false
for ; i < len(x.blocks) && x.blocks[i].Start < end; i++ {
b := x.blocks[i]
lo, hi := max(b.Start, start), min(b.End, end)
if hi <= lo {
continue
}
- quote := CleanQuote(string(x.runes[lo:hi]))
+ if lo > covered && normalizeMarkdown(fullQuote(string(x.runes[covered:lo]))).text != "" {
+ partial = true
+ }
+ covered = max(covered, hi)
+ quote := fullQuote(string(x.runes[lo:hi]))
if quote == "" && b.Locator.Type != types.SourceLocatorPDF {
continue
}
@@ -66,8 +68,13 @@ func (x *Index) Locators(start, end int) types.SourceLocators {
loc.Quote = quote
out = AppendLocator(out, loc)
}
- if len(out) > MaxLocatorsPerChunk {
- out = out[:MaxLocatorsPerChunk]
+ if covered < end && normalizeMarkdown(fullQuote(string(x.runes[covered:end]))).text != "" {
+ partial = true
+ }
+ if partial {
+ for i := range out {
+ out[i].Partial = true
+ }
}
return out
}
@@ -101,7 +108,11 @@ func AppendLocator(list types.SourceLocators, loc types.SourceLocator) types.Sou
}
func foldInto(last *types.SourceLocator, loc types.SourceLocator) bool {
- if last.Type != loc.Type {
+ if last.Type != loc.Type ||
+ last.Partial != loc.Partial ||
+ last.Mapping != loc.Mapping ||
+ last.SourceHash != loc.SourceHash ||
+ last.SourceID != loc.SourceID {
return false
}
switch loc.Type {
@@ -158,14 +169,19 @@ func MergeLocators(a, b types.SourceLocators) types.SourceLocators {
out = append(out, loc)
}
}
- if len(out) > MaxLocatorsPerChunk {
- out = out[:MaxLocatorsPerChunk]
- }
return out
}
func sameLocator(a, b types.SourceLocator) bool {
- if a.Type != b.Type || a.Page != b.Page || a.Block != b.Block || a.Slide != b.Slide ||
+ if a.Partial != b.Partial ||
+ a.Quote != b.Quote ||
+ a.SourceID != b.SourceID ||
+ a.SourceHash != b.SourceHash ||
+ a.Mapping != b.Mapping ||
+ a.Type != b.Type ||
+ a.Page != b.Page ||
+ a.Block != b.Block ||
+ a.Slide != b.Slide ||
a.Sheet != b.Sheet || a.RowStart != b.RowStart || a.RowEnd != b.RowEnd ||
a.Start != b.Start || a.End != b.End || a.StartMs != b.StartMs || a.EndMs != b.EndMs ||
a.Section != b.Section || len(a.BBox) != len(b.BBox) {
@@ -190,21 +206,27 @@ var (
// text, collapses whitespace, and caps it at types.SourceLocatorQuoteMax
// runes.
func CleanQuote(text string) string {
+ return capRunes(fullQuote(text), types.SourceLocatorQuoteMax)
+}
+
+// fullQuote keeps the complete evidence; truncation can hide a citation near its end.
+func fullQuote(text string) string {
text = quoteImageRe.ReplaceAllString(text, " ")
text = quoteLinkRe.ReplaceAllString(text, "$1")
text = quoteTagRe.ReplaceAllString(text, " ")
+ text = html.UnescapeString(text)
text = strings.TrimSpace(quoteSpaceRe.ReplaceAllString(text, " "))
- return capRunes(text, types.SourceLocatorQuoteMax)
+ return text
}
func joinQuote(a, b string) string {
if a == "" {
return b
}
- if b == "" || utf8.RuneCountInString(a) >= types.SourceLocatorQuoteMax {
+ if b == "" {
return a
}
- return capRunes(a+" "+b, types.SourceLocatorQuoteMax)
+ return a + " " + b
}
func capRunes(s string, n int) string {
diff --git a/internal/sourceloc/precision_test.go b/internal/sourceloc/precision_test.go
new file mode 100644
index 00000000..76109235
--- /dev/null
+++ b/internal/sourceloc/precision_test.go
@@ -0,0 +1,177 @@
+package sourceloc
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAlignNeverAttributesMissingBlockToPreviousPage(t *testing.T) {
+ md := "第一段正确文本。\n第二段实际内容没有匹配。\n第三段正确文本。"
+ units := []Unit{
+ {Text: "第一段正确文本", Locator: types.SourceLocator{Type: "pdf", Page: 1}},
+ {Text: "第二段OCR错误内容", Locator: types.SourceLocator{Type: "pdf", Page: 2}},
+ {Text: "第三段正确文本", Locator: types.SourceLocator{Type: "pdf", Page: 3}},
+ }
+ index := NewIndex(md, Align(md, units))
+ at := runeIndex(md, "第二段")
+ require.Empty(t, index.Locators(at, at+5))
+ require.Equal(t, 3, index.Locators(runeIndex(md, "第三段"), len([]rune(md)))[0].Page)
+}
+
+func TestAlignRequiresFullUnitNotCommonPrefix(t *testing.T) {
+ prefix := strings.Repeat("共同前缀", 12)
+ require.Empty(t,
+ Align(prefix+"验收不合格",
+ []Unit{{
+ Text: prefix + "验收合格",
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 1,
+ },
+ }}))
+}
+
+func TestAlignPreservesNumericMeaning(t *testing.T) {
+ for _, pair := range [][2]string{{"数值15毫米", "数值1.5毫米"}, {"温度5度", "温度-5度"}, {"比例5", "比例5%"}, {"值≤5", "值≥5"}} {
+ require.Empty(t,
+ Align(pair[0],
+ []Unit{{
+ Text: pair[1],
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 1,
+ },
+ }}),
+ pair)
+ }
+}
+
+func TestAlignRejectsUnaccountedDuplicate(t *testing.T) {
+ require.Empty(t,
+ Align("目录:设备验收合格。\n正文:设备验收合格。",
+ []Unit{{
+ Text: "设备验收合格",
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 9,
+ },
+ }}))
+ units := []Unit{
+ {
+ Text: "相同段落内容",
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 1,
+ },
+ },
+ {
+ Text: "相同段落内容",
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 2,
+ },
+ },
+ }
+ blocks := Align("相同段落内容\n相同段落内容", units)
+ require.Len(t, blocks, 2)
+ require.Equal(t, 2, blocks[1].Locator.Page)
+}
+
+func TestRemapNeverInterpolatesChangedText(t *testing.T) {
+ old := "第一段\n已通过验收\n末段"
+ blocks := []types.SourceBlock{{Start: 4, End: 9, Locator: types.SourceLocator{Type: "pdf", Page: 2}}}
+ require.Empty(t, RemapBlocks(blocks, old, "第一段\n未通过检查\n末段"))
+}
+
+func TestMergeKeepsDistinctExcerptsOfSameRegion(t *testing.T) {
+ a := types.SourceLocator{Type: "pdf", Page: 2, BBox: []float64{0, 0, 1, 1}, Quote: "第一句"}
+ b := a
+ b.Quote = "另一句"
+ require.Len(t, MergeLocators(types.SourceLocators{a}, types.SourceLocators{a, b}), 2)
+}
+
+func TestLocatorsPreserveLongEvidenceAndAllRegions(t *testing.T) {
+ text := strings.Repeat("完整原文", 100) + "末尾引用的关键结论"
+ index := NewIndex(text,
+ []types.SourceBlock{{
+ Start: 0,
+ End: len([]rune(text)),
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: 1,
+ },
+ }})
+ require.Equal(t, text, index.Locators(0, len([]rune(text)))[0].Quote)
+ var blocks []types.SourceBlock
+ for i := 0; i < 80; i++ {
+ blocks = append(blocks,
+ types.SourceBlock{
+ Start: i,
+ End: i + 1,
+ Locator: types.SourceLocator{
+ Type: "pdf",
+ Page: i + 1,
+ },
+ })
+ }
+ locs := NewIndex(strings.Repeat("文", 80), blocks).Locators(0, 80)
+ require.Len(t, locs, 80)
+ require.Len(t, MergeLocators(locs[:40], locs[40:]), 80)
+}
+
+func TestFoldingCannotMixRevisionsOrMappingQuality(t *testing.T) {
+ first := types.SourceLocator{Type: "pdf", Page: 1, Mapping: "exact", SourceHash: "original", Quote: "第一段"}
+ second := first
+ second.Mapping = ""
+ require.Len(t, AppendLocator(types.SourceLocators{first}, second), 2)
+ second = first
+ second.SourceHash = "replacement"
+ require.Len(t, AppendLocator(types.SourceLocators{first}, second), 2)
+}
+
+func TestAlignRejectsPartialNumericValues(t *testing.T) {
+ for _, pair := range [][2]string{{"增长为5%", "增长为5"}, {"115", "15"}, {"15.5", "15"}, {"-15", "15"}} {
+ require.Empty(t,
+ Align(pair[0], []Unit{{Text: pair[1], Locator: types.SourceLocator{Type: "pdf", Page: 1}}}),
+ pair,
+ )
+ }
+}
+
+func TestChunkWithUnmappedEvidenceIsMarkedPartial(t *testing.T) {
+ md := "匹配的原文。\n未匹配的关键结论。"
+ index := NewIndex(md, Align(md, []Unit{{Text: "匹配的原文", Locator: types.SourceLocator{Type: "pdf", Page: 1}}}))
+ locs := index.Locators(0, len([]rune(md)))
+ require.Len(t, locs, 1)
+ require.True(t, locs[0].Partial)
+ require.False(t, index.Locators(0, 5)[0].Partial)
+}
+
+func TestAlignmentDecodesTableEntitiesWithoutLosingOffsets(t *testing.T) {
+ md := ""
+ loc := types.SourceLocator{Type: "pdf", Page: 2}
+ blocks := Align(md, []Unit{{Text: "限值 <1.5", Locator: loc}})
+ require.Len(t, blocks, 1)
+ quotes := NewIndex(md, blocks).Locators(0, len([]rune(md)))
+ require.False(t, quotes[0].Partial)
+ require.Equal(t, "限值 <1.5", quotes[0].Quote)
+ remapped := RemapBlocks(blocks, md, "|限值|<1.5|")
+ require.Len(t, remapped, 1)
+ require.Equal(t, 2, remapped[0].Locator.Page)
+}
+
+func TestImagePlaceholderDoesNotInvalidateVerifiedTextLocators(t *testing.T) {
+ md := "第一段完整的原文。\n\n\n\n第二段完整的原文。"
+ blocks := Align(md, []Unit{
+ {Text: "第一段完整的原文。", Locator: types.SourceLocator{Type: "docx", Block: 1}},
+ {Text: "第二段完整的原文。", Locator: types.SourceLocator{Type: "docx", Block: 3}},
+ })
+ locs := NewIndex(md, blocks).Locators(0, len([]rune(md)))
+ require.Len(t, locs, 2)
+ for _, loc := range locs {
+ require.False(t, loc.Partial)
+ }
+}
diff --git a/internal/sourceloc/remap.go b/internal/sourceloc/remap.go
index d67ff41a..be5e6a4a 100644
--- a/internal/sourceloc/remap.go
+++ b/internal/sourceloc/remap.go
@@ -28,7 +28,8 @@ type Remapper struct {
pair []int // pair[i] = new line index matched to old line i, or -1
}
-// NewRemapper prepares a mapping from oldText to newText.
+// NewRemapper prepares a best-effort offset mapping.
+// Deprecated: source positions must use RemapBlocks, which verifies full text.
func NewRemapper(oldText, newText string) *Remapper {
oldKeys, oldStarts, oldLen := splitLines(oldText)
newKeys, newStarts, newLen := splitLines(newText)
@@ -120,17 +121,18 @@ func RemapBlocks(blocks []types.SourceBlock, oldText, newText string) []types.So
if len(blocks) == 0 || oldText == newText {
return blocks
}
- r := NewRemapper(oldText, newText)
- out := make([]types.SourceBlock, 0, len(blocks))
+ // Re-align full block text after markup/URL rewrites. Proportional
+ // interpolation can manufacture a location for deleted or changed content.
+ oldRunes := []rune(oldText)
+ units := make([]Unit, 0, len(blocks))
for _, b := range blocks {
- start, end := r.Map(b.Start), r.Map(b.End)
- if end <= start {
+ if b.Start < 0 || b.End > len(oldRunes) || b.End <= b.Start {
continue
}
- b.Start, b.End = start, end
- out = append(out, b)
+ text := normalizeMarkdown(string(oldRunes[b.Start:b.End])).text
+ units = append(units, Unit{Text: text, Locator: b.Locator})
}
- return out
+ return Align(newText, units)
}
// splitLines returns each line's comparison key (without its line break or a
diff --git a/internal/sourceloc/text.go b/internal/sourceloc/text.go
index 7a04ee56..e40ef2df 100644
--- a/internal/sourceloc/text.go
+++ b/internal/sourceloc/text.go
@@ -14,7 +14,7 @@ func TextBlocks(content string) []types.SourceBlock {
if start >= 0 && end > start {
blocks = append(blocks, types.SourceBlock{
Start: start, End: end,
- Locator: types.SourceLocator{Type: types.SourceLocatorText, Start: start, End: end},
+ Locator: types.SourceLocator{Type: types.SourceLocatorText, Mapping: "exact", Start: start, End: end},
})
}
start = -1
diff --git a/internal/types/chunk.go b/internal/types/chunk.go
index cfa834cb..719e7ff7 100644
--- a/internal/types/chunk.go
+++ b/internal/types/chunk.go
@@ -84,6 +84,8 @@ func (f ChunkFlags) ToggleFlag(flag ChunkFlags) ChunkFlags {
// ImageInfo 表示与 Chunk 关联的图片信息
type ImageInfo struct {
+ // SHA256 identifies the exact stored image bytes across parser and preview.
+ SHA256 string `json:"sha256,omitempty"`
// 图片URL(COS)
URL string `json:"url" gorm:"type:text"`
// 原始图片URL
diff --git a/internal/types/search.go b/internal/types/search.go
index 3d3c0a9a..1ed0d586 100644
--- a/internal/types/search.go
+++ b/internal/types/search.go
@@ -149,6 +149,8 @@ func (st SearchTargets) ContainsKB(kbID string) bool {
// SearchResult represents the search result
type SearchResult struct {
+ // CitationSources retains independently citeable bodies after context expansion.
+ CitationSources []*SearchResult `json:"-" gorm:"-"`
// ID
ID string `gorm:"column:id" json:"id"`
// Content
diff --git a/internal/types/source_locator.go b/internal/types/source_locator.go
index 071f365e..b54d4c05 100644
--- a/internal/types/source_locator.go
+++ b/internal/types/source_locator.go
@@ -26,13 +26,21 @@ const (
SourceLocatorSection = "section"
)
-// SourceLocatorQuoteMax caps the quote stored per locator, in runes.
+// SourceLocatorQuoteMax caps display snippets, not stored source evidence.
const SourceLocatorQuoteMax = 300
// SourceLocator is one position in an original file. Only the fields of its
// Type are set; zero values are omitted from JSON and read back as zero.
type SourceLocator struct {
Type string `json:"type"`
+ // Mapping records verified text-to-source alignment. Empty means legacy.
+ Mapping string `json:"mapping,omitempty"`
+ // Partial means some of the chunk has no source mapping.
+ Partial bool `json:"partial,omitempty"`
+ // SourceID identifies a block within the parser result, not a rendered item.
+ SourceID string `json:"source_id,omitempty"`
+ // SourceHash binds the locator to the knowledge record’s source revision hash.
+ SourceHash string `json:"source_hash,omitempty"`
// Page is the 1-based PDF page.
Page int `json:"page,omitempty"`
// BBox is [x0, y0, x1, y1] as fractions of the page width and height,
@@ -57,8 +65,8 @@ type SourceLocator struct {
Section int `json:"section,omitempty"`
// Title names the section or slide when the source has one.
Title string `json:"title,omitempty"`
- // Quote is the cited text as parsed, capped at SourceLocatorQuoteMax
- // runes. Viewers match it inside the rendered original and against the
+ // Quote is the complete cited overlap as parsed. Viewers match it
+ // inside the rendered original and against the
// answer sentence that cites the chunk.
Quote string `json:"quote,omitempty"`
}