diff --git a/internal/infrastructure/docparser/anydoc/convert_linked_test.go b/internal/infrastructure/docparser/anydoc/convert_linked_test.go index ba95816d..daf8512f 100644 --- a/internal/infrastructure/docparser/anydoc/convert_linked_test.go +++ b/internal/infrastructure/docparser/anydoc/convert_linked_test.go @@ -10,6 +10,7 @@ import ( "sync" "sync/atomic" "testing" + "time" ) // These tests exercise the linked Rust converter, so they only build with the @@ -191,6 +192,36 @@ func TestDeeplyNestedPDFFailsWithoutKillingTheProcess(t *testing.T) { } } +// A show-text operator with no operand made the page-classification scan walk +// back over the whole content stream looking for one, so a stream of bare +// `] TJ` tokens cost time quadratic in its length: before pdf-inspector 1.14.2 +// this input took 26 seconds of CPU, and a 2 MB one took nearly two minutes. +// Unbounded work is the half of the hostile-PDF problem that guarded() cannot +// catch — it never panics, it just holds the core — so the bound is pinned +// here. The budget is deliberately far above the ~7ms a bounded lookback needs: +// what it has to distinguish is linear from quadratic, not fast from slow. +func TestDetectorLookbackStaysLinear(t *testing.T) { + const ( + operators = 200000 + budget = 15 * time.Second + ) + + done := make(chan struct{}) + go func() { + defer close(done) + // The result is irrelevant: the page carries one real text run, so + // this converts either way. Only how long it takes is under test. + _, _ = Convert(unmatchedShowTextPDF(operators), Options{Format: "pdf"}) + }() + + select { + case <-done: + case <-time.After(budget): + t.Fatalf("converting a PDF with %d unmatched TJ operators took over %s; "+ + "the detector's operand lookback is no longer bounded", operators, budget) + } +} + // buildDocx writes the smallest OOXML package that carries a heading, a // paragraph, and one embedded image. func buildDocx(t *testing.T) []byte { @@ -304,6 +335,21 @@ func nestedPDF(nested string) []byte { }) } +// unmatchedShowTextPDF is a one-page PDF whose content stream shows text once +// and then repeats a TJ operator whose array operand is never opened. +func unmatchedShowTextPDF(operators int) []byte { + content := "BT /F1 12 Tf 20 100 Td (Shipping summary) Tj ET\n" + + strings.Repeat("] TJ\n", operators) + return writePDF([]string{ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R " + + "/Resources << /Font << /F1 5 0 R >> >> >>", + fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(content), content), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + }) +} + // textlessPDF is a one-page PDF with no text stream: the shape of a scanned // exam paper as far as pdf-inspector is concerned. func textlessPDF() []byte { diff --git a/scripts/build-anydoc-lib.sh b/scripts/build-anydoc-lib.sh index ed70478a..2a25d010 100755 --- a/scripts/build-anydoc-lib.sh +++ b/scripts/build-anydoc-lib.sh @@ -41,31 +41,42 @@ case "$target" in *) lib_name=libanydoc_go.a ;; esac -# document_to_markdown is private in published anydoc. Copy 0.1.8 and re-export -# that one function so anydoc-go can keep the official serializer. +# The crate version to patch is read from Cargo.toml rather than repeated here, +# so an upgrade is a one-line change to the `anydoc = "=X.Y.Z"` pin. +anydoc_version=$(sed -n 's/^anydoc = "=\([0-9][^"]*\)".*/\1/p' "$crate_dir/Cargo.toml" | head -1) +if [ -z "$anydoc_version" ]; then + echo "error: no pinned 'anydoc = \"=X.Y.Z\"' dependency in $crate_dir/Cargo.toml" >&2 + exit 1 +fi + +# document_to_markdown is private in published anydoc. Copy the pinned version +# and re-export that one function so anydoc-go can keep the official serializer. prepare_patched_anydoc() { local dest="$crate_dir/patched-anydoc" - if [ -f "$dest/.weknora-patched" ]; then + # The marker records which version was patched: after a version bump the + # copy left by the previous build is the wrong crate, and reusing it would + # fail deep inside cargo's patch resolution instead of here. + if [ -f "$dest/.weknora-patched" ] && [ "$(cat "$dest/.weknora-patched")" = "$anydoc_version" ]; then return fi local src="" local cargo_home="${CARGO_HOME:-$HOME/.cargo}" - src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name 'anydoc-0.1.8' 2>/dev/null | head -1 || true) + src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name "anydoc-$anydoc_version" 2>/dev/null | head -1 || true) if [ -z "$src" ]; then - local tarball="$crate_dir/.anydoc-0.1.8.crate" - echo "Fetching anydoc 0.1.8 to patch document_to_markdown into the public API" - if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/0.1.8/download" -o "$tarball"; then - curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-0.1.8.crate" -o "$tarball" + local tarball="$crate_dir/.anydoc-$anydoc_version.crate" + echo "Fetching anydoc $anydoc_version to patch document_to_markdown into the public API" + if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/$anydoc_version/download" -o "$tarball"; then + curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-$anydoc_version.crate" -o "$tarball" fi local unpack="$crate_dir/.anydoc-unpack" rm -rf "$unpack" mkdir -p "$unpack" tar -xzf "$tarball" -C "$unpack" - src=$(find "$unpack" -maxdepth 1 -type d -name 'anydoc-0.1.8' | head -1) + src=$(find "$unpack" -maxdepth 1 -type d -name "anydoc-$anydoc_version" | head -1) if [ -z "$src" ]; then - echo "error: unpacked anydoc-0.1.8 crate is missing" >&2 + echo "error: unpacked anydoc-$anydoc_version crate is missing" >&2 exit 1 fi fi @@ -73,20 +84,30 @@ prepare_patched_anydoc() { rm -rf "$dest" cp -R "$src" "$dest" if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then - echo "error: anydoc 0.1.8 lib.rs no longer has the expected document_to_markdown import" >&2 + echo "error: anydoc $anydoc_version lib.rs no longer has the expected document_to_markdown import" >&2 exit 1 fi sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs" rm -f "$dest/src/lib.rs.bak" - touch "$dest/.weknora-patched" - rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-0.1.8.crate" + printf '%s\n' "$anydoc_version" > "$dest/.weknora-patched" + rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-$anydoc_version.crate" } +# version.go is what the parser reports as `anydoc_version` on every parsed +# document, so a bump that misses it would mislabel the whole knowledge base. +go_version=$(sed -n 's/^const Version = "\([^"]*\)".*/\1/p' "$crate_dir/version.go" | head -1) +if [ "$go_version" != "$anydoc_version" ]; then + echo "error: version.go says '$go_version' but Cargo.toml pins anydoc '$anydoc_version'." >&2 + echo "Bump both: the Go constant is the version WeKnora records for parsed documents." >&2 + exit 1 +fi + +echo "Building anydoc $anydoc_version archive for $target" +prepare_patched_anydoc + # --locked: build exactly the dependency versions in the committed Cargo.lock. # That lockfile is what pins the patched pdf-inspector/lopdf, so a silent # resolver drift must fail the build rather than ship an unaudited tree. -echo "Building anydoc archive for $target" -prepare_patched_anydoc cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target" dest="$crate_dir/lib/$lib_dir" diff --git a/third_party/anydoc-go/Cargo.lock b/third_party/anydoc-go/Cargo.lock index 4425fa1a..b33122f5 100644 --- a/third_party/anydoc-go/Cargo.lock +++ b/third_party/anydoc-go/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "anydoc" -version = "0.1.8" +version = "0.1.9" dependencies = [ "calamine", "cfb", @@ -104,7 +104,7 @@ dependencies = [ [[package]] name = "anydoc-go" -version = "0.1.8" +version = "0.1.9" dependencies = [ "anydoc", "cbindgen", @@ -868,9 +868,9 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdf-inspector" -version = "0.1.8" +version = "1.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +checksum = "1e024ae242c514e2adf6aee186678e0eabdc2e5ecfbb2159186881b4498593cb" dependencies = [ "env_logger", "include_dir", diff --git a/third_party/anydoc-go/Cargo.toml b/third_party/anydoc-go/Cargo.toml index 433f6961..945ef5ff 100644 --- a/third_party/anydoc-go/Cargo.toml +++ b/third_party/anydoc-go/Cargo.toml @@ -5,7 +5,7 @@ # symbols from, not a loadable shared object. [package] name = "anydoc-go" -version = "0.1.8" +version = "0.1.9" edition = "2024" description = "Go bindings (C ABI) for anydoc" license = "MIT" @@ -19,14 +19,14 @@ crate-type = ["staticlib"] # workspace, so it takes the published crate rather than a path dependency. # Pinned exactly: the C ABI mirrors this version's error and format enums. [dependencies] -anydoc = "=0.1.8" +anydoc = "=0.1.9" [build-dependencies] cbindgen = "0.29" # document_to_markdown is crate-private in published anydoc. The build script -# copies 0.1.8 and re-exports that one function so we can keep the official -# GFM serializer after rewriting asset images to External URLs. +# copies the pinned version above and re-exports that one function so we can +# keep the official GFM serializer after rewriting asset images to External URLs. [patch.crates-io] anydoc = { path = "patched-anydoc" } diff --git a/third_party/anydoc-go/README.md b/third_party/anydoc-go/README.md index 8eff10eb..7448e025 100644 --- a/third_party/anydoc-go/README.md +++ b/third_party/anydoc-go/README.md @@ -29,7 +29,7 @@ only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it. | Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") | | PR head | `1a7a6c0` | | Rebased onto | `4e3089b` (`chore: release v0.1.8`) | -| anydoc crate | `0.1.8` (from crates.io, pinned with `=`) | +| anydoc crate | `0.1.9` (from crates.io, pinned with `=`) | | License | MIT (see LICENSE) | The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before @@ -40,12 +40,73 @@ version, bumped from 0.1.3 to 0.1.8. Taking v0.1.8 also picks up the `pdf-inspector` 0.1.8 bump that fixes [RUSTSEC-2026-0187](https://rustsec.org/advisories/RUSTSEC-2026-0187.html), the `lopdf` stack overflow that aborts the process on a hostile PDF. +The pin then moved to 0.1.9, which is that same binding source against a newer +PDF stack: anydoc's `src/` is byte-identical between the two releases, so the C +ABI, the document model, and the GFM serializer this binding reaches into are +unchanged, and none of the local modifications below had to be re-applied. What +0.1.9 does change is `pdf-inspector`, from the crate published as 0.1.8 to +1.14.2 (the jump in the number is that project unifying its Rust, Python, and +Node versions, not fourteen major releases). See "Why 0.1.9 matters here" below. + +## Why 0.1.9 matters here + +anydoc's changelog lists 0.1.9 as a single dependency bump, which reads like +housekeeping. It is nine `pdf-inspector` fixes, and seven of them are +denial-of-service bounds on PDF parsing. + +Those seven bound work that had no bound at all: Form XObject expansion per +page, CID `/W` ranges, Encoding `cidrange` and ToUnicode `bfrange` expansion, +content-stream decode before operators are allocated, the detector's `Tj`/`TJ` +operand lookback, and disjoint-rect table clustering. Every one sits on the path +`pdf_inspector::process_pdf_mem` takes — the only way anydoc converts a PDF — +and every one is driven by nothing but the bytes of the uploaded file. + +This is the class of bug the `guarded()` panic catcher explicitly cannot +contain, for the same reason `RUSTSEC-2026-0187` could not be: unbounded CPU +never panics, and an allocation failure aborts. The detector lookback is the +cheapest to demonstrate — before 1.14.2, each `Tj`/`TJ` whose operand was +missing rescanned the whole content stream, so a stream of bare `] TJ` tokens +was quadratic: + +| `] TJ` tokens | PDF size | 0.1.8 | 0.1.9 | +| --- | --- | --- | --- | +| 40,000 | 195 KB | 1.0 s | 2.0 ms | +| 120,000 | 586 KB | 9.7 s | 4.1 ms | +| 200,000 | 977 KB | 25.9 s | 6.5 ms | +| 400,000 | 1.9 MB | 105.6 s | 12.2 ms | + +One run on a 4-core VM, same Go code either side, only the linked archive +differing. Quadrupling the input cost 16× the time before the fix and roughly +4× after. A 2 MB upload holding a core for nearly two minutes needs no +privilege and no malformed container — the file parses fine, it just takes +forever, and every concurrent upload takes a core of its own. +`TestDetectorLookbackStaysLinear` in `internal/infrastructure/docparser/anydoc` +fails if a bump reintroduces it. + +Two commits improve extraction rather than bound it, and both change what gets +indexed: text inside Form XObjects now tracks the text line matrix and honors +`T*`, `TL`, `'`, `"`, `Tc`, and `Tw`, so nested form text keeps its line breaks +and spacing instead of running together, and small-caps runs merge instead of +being read as extra table columns — a spurious column reaches the model as a +misaligned markdown table. + +Nothing else in the range reaches WeKnora, which is worth recording so the next +upgrade does not go looking for it. The crate published as 0.1.8 was cut at +`pdf-inspector`'s `packages-2026-08-10` tag, one release behind 1.14.1, so 0.1.9 +formally spans that release too — but its two functional commits both miss this +binding. One only touches the Node and Python surfaces. The other serves +invisible (`Tr 3`) OCR text layers instead of reporting `needs_ocr`, which +sounds like it would spare WeKnora an OCR round trip, except that it landed in +`extract_text_in_regions`; anydoc uses the positioned-text path, which already +retried with invisible text included. Scanned PDFs still come back as "OCR is +required" and still fall back to the docreader. + ## Local modifications Keep this list current: it is the diff a future upgrade has to re-apply. Items 2–4 are bugs in the upstream PR and are worth sending back to it. -1. `Cargo.toml` — depends on the published `anydoc = "=0.1.8"` crate instead of +1. `Cargo.toml` — depends on the published `anydoc = "=0.1.9"` crate instead of the workspace path dependency, declares its own empty `[workspace]`, and repeats the upstream release profile (`lto`, `strip`), which it would otherwise inherit from the anydoc workspace. @@ -72,9 +133,12 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Items 5. Removed the upstream CLI (`cmd/anydoc`) and the binding test suite, which reads fixtures from the anydoc repository. WeKnora's own tests live in `internal/infrastructure/docparser/anydoc`. -6. `scripts/build-anydoc-lib.sh` copies anydoc 0.1.8 to `patched-anydoc/` - (gitignored) and re-exports `document_to_markdown`, which is crate-private - in the published crate. `src/asset_links.rs` then rewrites `ImageSource::Asset` +6. `scripts/build-anydoc-lib.sh` copies the pinned anydoc release to + `patched-anydoc/` (gitignored) and re-exports `document_to_markdown`, which + is crate-private in the published crate. It reads the version from the + `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a bump + is one line and cannot half-land. + `src/asset_links.rs` then rewrites `ImageSource::Asset` to `External("images/image-N.ext")` so the official serializer emits in-place image links. `anydoc_to_markdown_with_asset_links` is the ABI for that path. @@ -92,6 +156,11 @@ kills the process with a stack overflow (`RUSTSEC-2026-0187`), which neither and the same input comes back as an ordinary error; `TestDeeplyNestedPDFFailsWithoutKillingTheProcess` keeps it that way. +Moving the pin to 0.1.9 added no crate and removed none: `lopdf` stays at 0.42 +and the whole tree below `pdf-inspector` is unchanged, so the lockfile diff is +three version lines and one checksum. The audit result is therefore the same +one described below. + `cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is unmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. It is not a vulnerability and nothing here can fix it, so warnings report without diff --git a/third_party/anydoc-go/version.go b/third_party/anydoc-go/version.go index 1d5bfb3c..e493d82f 100644 --- a/third_party/anydoc-go/version.go +++ b/third_party/anydoc-go/version.go @@ -3,4 +3,4 @@ package anydoc // Version is the version of the anydoc-go module, bumped in lockstep with the // Rust crate version (go/Cargo.toml) and the GitHub release tag. The // packaged static libraries. -const Version = "0.1.8" +const Version = "0.1.9"