diff --git a/.air.toml b/.air.toml index d471342e..5a4065be 100644 --- a/.air.toml +++ b/.air.toml @@ -5,7 +5,7 @@ tmp_dir = "tmp" [build] args_bin = [] bin = "./tmp/main" - cmd = "go build -ldflags=\"$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'\" -o ./tmp/main ./cmd/server" + cmd = "go build -tags \"${GO_BUILD_TAGS}\" -ldflags=\"$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'\" -o ./tmp/main ./cmd/server" delay = 1000 exclude_dir = ["assets", "tmp", "vendor", "testdata", "frontend", "migrations", "node_modules", "docs", "logs", "data", "WeKnora-Chrome-Extension", "Formula", "dataset", "docreader", "docker", "llm_debug", "web", "mcp-server", "misc", "scripts", "dist", "client", "helm", "website-docs"] exclude_file = [] diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml index 1f7bce20..0c138bc0 100644 --- a/.github/workflows/app.yml +++ b/.github/workflows/app.yml @@ -14,6 +14,7 @@ on: - "scripts/**" - "skills/preloaded/**" - "docker/Dockerfile.app" + - "third_party/anydoc-go/**" - ".github/workflows/app.yml" pull_request: paths: @@ -27,6 +28,7 @@ on: - "scripts/**" - "skills/preloaded/**" - "docker/Dockerfile.app" + - "third_party/anydoc-go/**" - ".github/workflows/app.yml" workflow_dispatch: @@ -96,3 +98,49 @@ jobs: - name: Build server run: go build ./cmd/server + + anydoc: + name: anydoc parser engine + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo-audit + id: cargo-audit-cache + uses: actions/cache@v4 + with: + path: ~/.cargo/bin/cargo-audit + key: cargo-audit-${{ runner.os }} + + - name: Install cargo-audit + if: steps.cargo-audit-cache.outputs.cache-hit != 'true' + run: cargo install cargo-audit --locked + + # The vendored bindings pull anydoc and its PDF stack from crates.io, and + # WeKnora parses untrusted uploads in-process. A Rust advisory there (as + # in RUSTSEC-2026-0187, an uncatchable abort on a hostile PDF) has to + # break the build, not wait for someone to notice. Vulnerabilities fail + # the job; unmaintained-crate warnings only report, since a transitive + # crate going unmaintained is not something this repo can act on. + - name: Audit the vendored dependency tree + working-directory: third_party/anydoc-go + run: cargo audit + + # The archive is a build artifact, not a committed blob, so CI is what + # keeps the vendored bindings in third_party/anydoc-go compiling. + - name: Build the anydoc static archive + run: ./scripts/build-anydoc-lib.sh + + - name: Vet, test, and build with the engine linked in + run: | + go vet -tags anydoc ./internal/infrastructure/docparser/... + go test -tags anydoc -count=1 ./internal/infrastructure/docparser/... + go build -tags anydoc ./cmd/server diff --git a/Makefile b/Makefile index acad8462..44b6598e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite +.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform dev-start dev-stop dev-restart dev-logs dev-status dev-app dev-frontend docs install-swagger build-lite run-lite package-lite anydoc-lib build-anydoc # Show help help: @@ -8,6 +8,8 @@ help: @echo " build 构建应用" @echo " run 运行应用" @echo " test 运行测试" + @echo " anydoc-lib 构建 anydoc 静态库(需要 Rust 工具链)" + @echo " build-anydoc 构建带 anydoc 解析引擎的应用" @echo " clean 清理构建文件" @echo "" @echo "Docker 命令:" @@ -56,6 +58,7 @@ help: @echo " dev-logs 查看开发环境日志" @echo " dev-status 查看开发环境状态" @echo " dev-app 启动后端应用(本地运行,需先运行 dev-start)" + @echo " 已 make anydoc-lib 时自动链接 anydoc 引擎" @echo " dev-frontend 启动前端(本地运行,需先运行 dev-start)" @echo "" @echo "Lite 模式(零外部依赖):" @@ -87,6 +90,15 @@ endif build: go build -o $(BINARY_NAME) $(MAIN_PATH) +# Build the anydoc static archive (Rust) that the `anydoc` build tag links. +# Override the platform with TARGET=. +anydoc-lib: + ./scripts/build-anydoc-lib.sh + +# Build the application with the in-process anydoc parser engine linked in. +build-anydoc: anydoc-lib + go build -tags anydoc -o $(BINARY_NAME) $(MAIN_PATH) + # Run the application run: build ./$(BINARY_NAME) @@ -234,6 +246,8 @@ deps: # Build for production # google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn for qdrant milvus proto conflict +# GO_BUILD_TAGS adds optional build tags, e.g. GO_BUILD_TAGS=anydoc to link the +# in-process office document parser (run `make anydoc-lib` first). build-prod: VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \ COMMIT_ID=$${COMMIT_ID:-unknown}; \ @@ -243,7 +257,7 @@ build-prod: BUILD_TIME=$${BUILD_TIME:-unknown}; \ GO_VERSION=$${GO_VERSION:-unknown}; \ LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.Edition=standard' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \ - go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH) + go build -tags "$(GO_BUILD_TAGS)" -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH) # Build Lite version (single binary, SQLite + in-memory queue) # 会先构建前端到 web/,再构建 Go 二进制;SKIP_FRONTEND=1 可跳过前端 diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 38e12b06..f2bf1b43 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -43,8 +43,27 @@ ENV COMMIT_ID=${COMMIT_ID_ARG} ENV BUILD_TIME=${BUILD_TIME_ARG} ENV GO_VERSION=${GO_VERSION_ARG} +# Optional: link the anydoc parser engine, which converts office documents in +# the Go process instead of the Python docreader. It is off by default because +# it needs a Rust toolchain (a few minutes and ~1 GB of build-stage layers) for +# an engine most deployments do not select. +# docker build --build-arg WITH_ANYDOC=1 ... +ARG WITH_ANYDOC=0 +ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo +ENV PATH=/usr/local/cargo/bin:$PATH +RUN if [ "$WITH_ANYDOC" = "1" ]; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain stable && \ + ./scripts/build-anydoc-lib.sh; \ + fi + # Build the application with version info -RUN --mount=type=cache,target=/go/pkg/mod make build-prod +RUN --mount=type=cache,target=/go/pkg/mod \ + if [ "$WITH_ANYDOC" = "1" ]; then \ + make build-prod GO_BUILD_TAGS=anydoc; \ + else \ + make build-prod; \ + fi RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/ # Final stage diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 37e870ae..681177e9 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -4976,6 +4976,10 @@ export default { name: 'Simple', desc: 'Simple format & image parsing (no external service required)' }, + anydoc: { + name: 'anydoc', + desc: 'In-process office document parser (no external service required)' + }, mineru: { name: 'MinerU', desc: 'MinerU self-hosted service' diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 6fbf69d9..08a25b53 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -1399,6 +1399,10 @@ export default { name: 'Simple', desc: '간단한 형식 및 이미지 파싱 (외부 서비스 불필요)' }, + anydoc: { + name: 'anydoc', + desc: '프로세스 내 오피스 문서 파싱 (외부 서비스 불필요)' + }, builtin: { name: '내장', desc: 'DocReader 내장 파서 엔진 (docx/pdf/xlsx 등 복잡한 형식)' diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 228b28df..5f2b0929 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -1399,6 +1399,10 @@ export default { name: 'Simple', desc: 'Простой формат и анализ изображений (внешний сервис не требуется)' }, + anydoc: { + name: 'anydoc', + desc: 'Разбор офисных документов внутри процесса (внешний сервис не требуется)' + }, builtin: { name: 'Встроенный', desc: 'Встроенный парсер DocReader (docx/pdf/xlsx и другие сложные форматы)' diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index d3d9a1b9..6491dd6e 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1401,6 +1401,10 @@ export default { name: 'Simple', desc: '简单格式 & 图片解析(无需外部服务)' }, + anydoc: { + name: 'anydoc', + desc: '进程内 Office 文档解析(无需外部服务)' + }, builtin: { name: '内置', desc: 'DocReader 内置解析引擎(docx/pdf/xlsx 等复杂格式)' diff --git a/frontend/src/views/settings/ParserEngineSettings.vue b/frontend/src/views/settings/ParserEngineSettings.vue index 8024be50..d0c6739c 100644 --- a/frontend/src/views/settings/ParserEngineSettings.vue +++ b/frontend/src/views/settings/ParserEngineSettings.vue @@ -478,11 +478,12 @@ const ENGINE_ORDER: Record = { builtin: 0, weknoracloud: 1, simple: 2, - markitdown: 3, - mineru: 4, - mineru_cloud: 5, - paddleocr_vl: 6, - paddleocr_vl_cloud: 7, + anydoc: 3, + markitdown: 4, + mineru: 5, + mineru_cloud: 6, + paddleocr_vl: 7, + paddleocr_vl_cloud: 8, } const sortedEngines = computed(() => { diff --git a/go.mod b/go.mod index e5b7d221..150a3fb3 100644 --- a/go.mod +++ b/go.mod @@ -152,6 +152,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/firecrawl/anydoc/go v0.1.8 github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -348,3 +349,7 @@ require ( ) replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 + +// The anydoc Go bindings are not published yet (firecrawl/anydoc#30 is still +// open), so they are vendored. Drop this replace once upstream tags go/vX.Y.Z. +replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 97c3a4b1..3ac25784 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3752,41 +3752,23 @@ func isLikelyRateLimitError(err error) bool { return false } -// Returns nil when the required service is unavailable. -func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { - switch engine { - case docparser.SimpleEngineName: - return &docparser.SimpleFormatReader{} - case docparser.WeKnoraCloudEngineName: - creds := s.tenantService.GetWeKnoraCloudCredentials(ctx) - if creds == nil { - logger.Warnf(ctx, "[resolveDocReader] WeKnoraCloud: no tenant credentials (fileType=%s)", fileType) - return nil - } - reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(creds.AppID, creds.AppSecret) - if err != nil { - logger.Errorf(ctx, "[resolveDocReader] WeKnoraCloud reader init failed: %v", err) - return nil - } - return reader - case "mineru": - return docparser.NewMinerUReader(overrides) - case "mineru_cloud": - return docparser.NewMinerUCloudReader(overrides) - case "paddleocr_vl": - return docparser.NewPaddleOCRVLReader(overrides) - case "paddleocr_vl_cloud": - return docparser.NewPaddleOCRVLCloudReader(overrides) - case "builtin": - // 明确指定使用 builtin 引擎(docreader),不使用 simple format 兜底 - return s.documentReader - default: - // 未指定引擎时的兜底逻辑:simple format 使用 Go 原生处理,其他使用 docreader - if !isURL && docparser.IsSimpleFormat(fileType) { - return &docparser.SimpleFormatReader{} - } - return s.documentReader +// resolveDocReader picks the reader for one parse request. The engine catalog +// itself lives in the docparser registry; this only supplies the dependencies +// the service owns. Returns nil when the chosen engine cannot run — an +// unconfigured cloud engine, a disconnected docreader — after logging why. +func (s *knowledgeService) resolveDocReader( + ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string, +) interfaces.DocReader { + reader, err := docparser.NewReader(ctx, engine, fileType, isURL, docparser.ReaderDeps{ + Overrides: overrides, + Remote: s.documentReader, + WeKnoraCloudCredentials: s.tenantService.GetWeKnoraCloudCredentials, + }) + if err != nil { + logger.Warnf(ctx, "[resolveDocReader] engine=%q fileType=%q unusable: %v", engine, fileType, err) + return nil } + return reader } // failKnowledge marks knowledge as failed (only on last retry) and returns an error. diff --git a/internal/application/service/temporary_document.go b/internal/application/service/temporary_document.go index ad1062ae..4cbfcc04 100644 --- a/internal/application/service/temporary_document.go +++ b/internal/application/service/temporary_document.go @@ -394,14 +394,15 @@ func (s *temporaryDocumentService) parse(ctx context.Context, document *types.Te if tenant, ok := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil { request.ParserEngineOverrides = tenant.ParserEngineConfig.ToOverridesMap() } - var result *types.ReadResult - if docparser.IsSimpleFormat(ext) && (request.ParserEngine == "" || request.ParserEngine == "auto") { - result, err = (&docparser.SimpleFormatReader{}).Read(ctx, request) - } else if s.documentReader != nil { - result, err = s.documentReader.Read(ctx, request) - } else { - err = fmt.Errorf("document reader is not configured") + deps := docparser.ReaderDeps{Overrides: request.ParserEngineOverrides, Remote: s.documentReader} + if s.tenantService != nil { + deps.WeKnoraCloudCredentials = s.tenantService.GetWeKnoraCloudCredentials } + reader, err := docparser.NewReader(ctx, parserEngine, strings.TrimPrefix(ext, "."), false, deps) + if err != nil { + return "", nil, nil, fmt.Errorf("parse document: %w", err) + } + result, err := reader.Read(ctx, request) if err != nil { return "", nil, nil, fmt.Errorf("parse document: %w", err) } diff --git a/internal/handler/session/attachment_processor.go b/internal/handler/session/attachment_processor.go index 3e40beaf..358b4219 100644 --- a/internal/handler/session/attachment_processor.go +++ b/internal/handler/session/attachment_processor.go @@ -244,16 +244,31 @@ func (p *AttachmentProcessor) processWithDocumentReader( parserEngine = s } } + overrides := getParserEngineOverridesFromContext(ctx) - result, err := p.documentReader.Read(ctx, &types.ReadRequest{ + // Engines that parse in this process (anydoc, MinerU, ...) are resolved + // through the registry so a chat attachment honours the same engine rules + // as an ingested document. Anything the registry cannot build here — a + // cloud engine whose credentials this path cannot resolve — falls back to + // the docreader, which is where every engine name went before. + reader, err := docparser.NewReader(ctx, parserEngine, normalizedType, false, docparser.ReaderDeps{ + Overrides: overrides, + Remote: p.documentReader, + }) + if err != nil { + logger.Warnf(ctx, "parser engine %q unusable for this attachment, using docreader: %v", parserEngine, err) + reader = p.documentReader + } + + result, err := reader.Read(ctx, &types.ReadRequest{ FileContent: data, FileName: fileName, FileType: normalizedType, ParserEngine: parserEngine, - ParserEngineOverrides: getParserEngineOverridesFromContext(ctx), + ParserEngineOverrides: overrides, }) if err != nil { - return fmt.Errorf("DocumentReader failed: %w", err) + return fmt.Errorf("document parsing failed: %w", err) } // Resolve embedded image refs to storage URLs. diff --git a/internal/infrastructure/docparser/anydoc/anydoc.go b/internal/infrastructure/docparser/anydoc/anydoc.go new file mode 100644 index 00000000..5965f26e --- /dev/null +++ b/internal/infrastructure/docparser/anydoc/anydoc.go @@ -0,0 +1,176 @@ +// Package anydoc converts office documents to Markdown inside the Go process, +// without the Python docreader service. +// +// The conversion itself is the anydoc Rust library, linked as a static archive +// through cgo (see third_party/anydoc-go). That archive needs a Rust toolchain +// to produce, so it is opt-in: only builds tagged `anydoc` link it. Every other +// build compiles the stub in backend_stub.go, where Available reports false and +// the engine registry hides the engine with a reason the UI can show. +// +// The exported surface is deliberately narrow — bytes in, Markdown and embedded +// images out — so the backend can be replaced (a WASI runtime, a future +// upstream Go module) without touching the parser code that calls it. +package anydoc + +import ( + "errors" + "fmt" + "path/filepath" + "slices" + "strings" +) + +// ImageDir is the markdown path prefix for extracted images. The image +// resolver matches references by this path and swaps them for storage URLs. +const ImageDir = "images/" + +// Result is one converted document. +type Result struct { + // Markdown is GitHub-Flavored Markdown for the whole document. + // When WithAssets is set, embedded images appear in place as + // `![alt](images/image-N.ext)` via anydoc's official serializer. + Markdown string + // Assets are the images embedded in the document, in document order. + // Always empty for PDF, which anydoc renders straight to Markdown + // without a document model. + Assets []Asset + // AssetsError explains why Assets is empty when images were asked for. + // The document-model parse is what both extracts images and places them + // in the markdown; when it fails, Markdown still comes from the text-only + // renderer so the conversion succeeds and the caller decides whether to + // log or ignore the loss. + AssetsError error +} + +// Asset is one image embedded in a document. +type Asset struct { + // ID is the document-model asset index, as referenced by in-place image + // sources. It is stable for a single conversion. + ID uint64 + // Name is a generated, extension-carrying file name ("image-1.png"): + // embedded assets have no name of their own inside the container. + Name string + // MediaType is the IANA media type the container declared. + MediaType string + // Data is the raw image bytes. + Data []byte + // Alt is the image's alternative text as the document author wrote it, + // empty when the document carries none. + Alt string + // Section is the text of the nearest heading above the image, empty when + // the image sits before the first heading. + Section string +} + +// Options tunes a single conversion. +type Options struct { + // Format names the parser explicitly ("docx", "csv", ...). Empty means + // the format is detected from the content, which every format except + // CSV carries a signature for. + Format string + // WithAssets extracts embedded images and renders them in place in the + // markdown using anydoc's official GFM serializer (asset images are + // rewritten to `images/image-N.ext` links first). Ignored for PDF. + WithAssets bool +} + +// ErrUnavailable is returned by Convert when the binding is not linked into +// this build. Callers that can fall back to another engine should check for it +// with errors.Is. +var ErrUnavailable = errors.New("anydoc: binding not built into this binary") + +// supportedExtensions are the file types anydoc converts, mapped to the format +// name its parser selector uses. The list is static so that engine metadata +// (file types shown in the UI) is identical whether or not the binding is +// linked in. +var supportedExtensions = map[string]string{ + "doc": "doc", + "docx": "docx", + "docm": "docx", + "odt": "odt", + "rtf": "rtf", + "ppt": "ppt", + "pptx": "pptx", + "pptm": "pptx", + "odp": "odp", + "xls": "xlsx", + "xlsx": "xlsx", + "xlsm": "xlsx", + "ods": "ods", + "epub": "epub", + "csv": "csv", + "pdf": "pdf", +} + +// SupportedFileTypes returns the extensions anydoc can convert, sorted so the +// engine list is stable across restarts. +func SupportedFileTypes() []string { + types := make([]string, 0, len(supportedExtensions)) + for ext := range supportedExtensions { + types = append(types, ext) + } + slices.Sort(types) + return types +} + +// FormatForFile resolves the anydoc format name for a file type or file name. +// ok is false for anything anydoc does not convert. +func FormatForFile(fileType, fileName string) (format string, ok bool) { + ext := normalizeExt(fileType) + if ext == "" { + ext = normalizeExt(filepath.Ext(fileName)) + } + format, ok = supportedExtensions[ext] + return format, ok +} + +// Supports reports whether anydoc converts this file type, regardless of +// whether the binding is linked into this build. +func Supports(fileType, fileName string) bool { + _, ok := FormatForFile(fileType, fileName) + return ok +} + +// Available reports whether conversions can actually run in this binary. +func Available() bool { return backendAvailable() } + +// UnavailableReason explains, for the UI, why Available is false. It returns +// "" when the binding is available. +func UnavailableReason() string { return backendUnavailableReason() } + +// Version reports the anydoc release the linked binding was built from, or "" +// when no binding is linked. +func Version() string { return backendVersion() } + +// Convert turns document bytes into Markdown. It returns ErrUnavailable when +// the binding is not linked into this build. +func Convert(data []byte, opts Options) (*Result, error) { + if len(data) == 0 { + return nil, fmt.Errorf("anydoc: empty document") + } + if !backendAvailable() { + return nil, fmt.Errorf("%w: %s", ErrUnavailable, backendUnavailableReason()) + } + // PDF has no document model, so asset extraction would only produce an + // error the caller has to special-case. Drop the request instead. + if opts.Format == "pdf" { + opts.WithAssets = false + } + return backendConvert(data, opts) +} + +// PDFNeedsOCR reports whether a Convert error means the PDF has no usable +// text layer. Scanned exam papers fail this way; the caller should fall back +// to an engine that can rasterize pages for OCR. +func PDFNeedsOCR(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "ocr is required") || + strings.Contains(msg, "no extractable text") +} + +func normalizeExt(s string) string { + return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(s), ".")) +} diff --git a/internal/infrastructure/docparser/anydoc/anydoc_test.go b/internal/infrastructure/docparser/anydoc/anydoc_test.go new file mode 100644 index 00000000..b54024e4 --- /dev/null +++ b/internal/infrastructure/docparser/anydoc/anydoc_test.go @@ -0,0 +1,84 @@ +package anydoc + +import ( + "errors" + "slices" + "testing" +) + +func TestFormatForFileResolvesTypeAndName(t *testing.T) { + cases := []struct { + name string + fileType string + fileName string + wantFormat string + wantOK bool + }{ + {name: "file type", fileType: "docx", wantFormat: "docx", wantOK: true}, + {name: "dotted file type", fileType: ".DOCX", wantFormat: "docx", wantOK: true}, + {name: "macro format maps to base format", fileType: "pptm", wantFormat: "pptx", wantOK: true}, + {name: "legacy excel maps to xlsx parser", fileType: "xls", wantFormat: "xlsx", wantOK: true}, + {name: "falls back to file name", fileName: "report.odt", wantFormat: "odt", wantOK: true}, + {name: "unsupported type", fileType: "png", wantOK: false}, + {name: "nothing to go on", wantOK: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + format, ok := FormatForFile(tc.fileType, tc.fileName) + if ok != tc.wantOK { + t.Fatalf("FormatForFile(%q, %q) ok = %v, want %v", tc.fileType, tc.fileName, ok, tc.wantOK) + } + if format != tc.wantFormat { + t.Fatalf("FormatForFile(%q, %q) = %q, want %q", tc.fileType, tc.fileName, format, tc.wantFormat) + } + }) + } +} + +// The engine list must look the same whether or not the converter is linked +// in, so that the UI can show the file types with a "not built in" reason. +func TestSupportedFileTypesIsStableAndSorted(t *testing.T) { + types := SupportedFileTypes() + if !slices.IsSorted(types) { + t.Fatalf("SupportedFileTypes is not sorted: %v", types) + } + for _, want := range []string{"docx", "pptx", "xlsx", "epub", "pdf", "csv"} { + if !slices.Contains(types, want) { + t.Errorf("SupportedFileTypes does not include %q: %v", want, types) + } + } +} + +func TestPDFNeedsOCR(t *testing.T) { + if PDFNeedsOCR(nil) { + t.Fatal("nil error was treated as needing OCR") + } + if !PDFNeedsOCR(errors.New("PDF has no extractable text (Scanned, 5 pages): OCR is required")) { + t.Error("scanned-PDF error was not detected") + } + if PDFNeedsOCR(errors.New("not a readable zip archive")) { + t.Error("unrelated error was treated as needing OCR") + } +} + +func TestConvertRejectsEmptyInput(t *testing.T) { + if _, err := Convert(nil, Options{Format: "docx"}); err == nil { + t.Fatal("Convert(nil) succeeded, want an error") + } +} + +// Without the build tag there is no converter, and callers must be able to +// detect that so they can fall back to another engine. +func TestConvertReportsUnavailableBackend(t *testing.T) { + if Available() { + t.Skip("converter is linked into this build") + } + _, err := Convert([]byte("id,name\n1,a\n"), Options{Format: "csv"}) + if !errors.Is(err, ErrUnavailable) { + t.Fatalf("Convert error = %v, want ErrUnavailable", err) + } + if UnavailableReason() == "" { + t.Error("UnavailableReason is empty while the converter is unavailable") + } +} diff --git a/internal/infrastructure/docparser/anydoc/backend_cgo.go b/internal/infrastructure/docparser/anydoc/backend_cgo.go new file mode 100644 index 00000000..4a104022 --- /dev/null +++ b/internal/infrastructure/docparser/anydoc/backend_cgo.go @@ -0,0 +1,207 @@ +//go:build anydoc && cgo + +package anydoc + +import ( + "fmt" + "mime" + "strings" + + upstream "github.com/firecrawl/anydoc/go" +) + +// This build links the anydoc Rust archive through the upstream cgo bindings +// (vendored under third_party/anydoc-go until they are published as a module). +// Conversion runs in-process: no subprocess, no HTTP hop, no Python. + +func backendAvailable() bool { return true } + +func backendUnavailableReason() string { return "" } + +func backendVersion() string { return upstream.Version } + +func backendConvert(data []byte, opts Options) (*Result, error) { + format, err := upstreamFormat(opts.Format) + if err != nil { + return nil, err + } + + if !opts.WithAssets { + markdown, err := upstream.ToMarkdownBytes(data, format) + if err != nil { + return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err) + } + return &Result{Markdown: markdown}, nil + } + + // Official GFM serializer after rewriting Asset images to External URLs + // (`images/image-N.ext`). ToDocument still supplies the image bytes. + document, err := upstream.ToDocument(data, format) + if err != nil { + markdown, mdErr := upstream.ToMarkdownBytes(data, format) + if mdErr != nil { + return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", mdErr) + } + return &Result{ + Markdown: markdown, + AssetsError: fmt.Errorf("anydoc: image extraction failed: %w", err), + }, nil + } + markdown, err := upstream.ToMarkdownWithAssetLinks(data, format) + if err != nil { + return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err) + } + return &Result{Markdown: markdown, Assets: collectAssets(document)}, nil +} + +// upstreamFormat maps our format name onto the binding's constant. An empty +// name means "detect from content". +func upstreamFormat(name string) (*upstream.Format, error) { + if name == "" { + return nil, nil + } + format, ok := upstream.FormatFromExtension(name) + if !ok { + return nil, fmt.Errorf("anydoc: unsupported format %q", name) + } + return &format, nil +} + +// collectAssets names each embedded image after its position in the document, +// because containers store assets without a usable file name of their own, and +// attaches the alt text and section the image appeared under. +func collectAssets(document *upstream.Document) []Asset { + if document == nil || len(document.Assets) == 0 { + return nil + } + placements := placeAssets(document.Blocks) + assets := make([]Asset, 0, len(document.Assets)) + for i, asset := range document.Assets { + if len(asset.Data) == 0 { + continue + } + placement := placements[uint64(i)] + assets = append(assets, Asset{ + ID: uint64(i), + Name: fmt.Sprintf("image-%d%s", i+1, extensionFor(asset.MediaType)), + MediaType: asset.MediaType, + Data: asset.Data, + Alt: placement.alt, + Section: placement.section, + }) + } + return assets +} + +// placement is where in the document an asset was referenced from. +type placement struct { + alt string + section string +} + +// placeAssets walks the document body and records, per asset, the alt text of +// the image that referenced it and the heading it sat under. The walk +// recurses into lists, tables, and quotes, since an image can be nested +// anywhere a paragraph can. +func placeAssets(blocks []upstream.Block) map[uint64]placement { + placements := make(map[uint64]placement) + section := "" + walkBlocks(blocks, §ion, placements) + return placements +} + +func walkBlocks(blocks []upstream.Block, section *string, placements map[uint64]placement) { + for _, block := range blocks { + switch block.Kind { + case "heading": + *section = inlineText(block.Content) + walkInlines(block.Content, *section, placements) + case "paragraph": + walkInlines(block.Content, *section, placements) + case "block_quote": + walkBlocks(block.Blocks, section, placements) + case "list": + if block.List != nil { + for _, item := range block.List.Items { + walkBlocks(item.Blocks, section, placements) + } + } + case "table": + if block.Table != nil { + for _, row := range block.Table.Grid { + for _, slot := range row { + if slot.Cell != nil { + walkBlocks(slot.Cell.Blocks, section, placements) + } + } + } + } + } + } +} + +func walkInlines(inlines []upstream.Inline, section string, placements map[uint64]placement) { + for _, inline := range inlines { + switch inline.Kind { + case "image": + if inline.Source == nil || inline.Source.Kind != "asset" || inline.Source.AssetID == nil { + continue + } + alt := "" + if inline.Alt != nil { + alt = strings.TrimSpace(*inline.Alt) + } + // First reference wins: the same asset can be placed more than + // once, and the earliest placement is the one a reader would + // associate it with. + if _, seen := placements[*inline.Source.AssetID]; !seen { + placements[*inline.Source.AssetID] = placement{alt: alt, section: section} + } + case "link": + walkInlines(inline.Content, section, placements) + } + } +} + +// inlineText flattens inline content to plain text, which is all a heading is +// needed for here. +func inlineText(inlines []upstream.Inline) string { + var text strings.Builder + for _, inline := range inlines { + switch inline.Kind { + case "text": + if inline.Text != nil { + text.WriteString(*inline.Text) + } + case "link": + text.WriteString(inlineText(inline.Content)) + } + } + return strings.TrimSpace(text.String()) +} + +func extensionFor(mediaType string) string { + switch strings.ToLower(strings.TrimSpace(mediaType)) { + case "image/jpeg", "image/jpg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + case "image/bmp": + return ".bmp" + case "image/tiff": + return ".tiff" + case "image/svg+xml": + return ".svg" + } + // Anything else (EMF/WMF drawings, unknown object payloads) keeps + // whatever extension the media type registry knows, and .bin otherwise, + // so downstream storage never writes an extension-less blob. + if exts, err := mime.ExtensionsByType(mediaType); err == nil && len(exts) > 0 { + return exts[0] + } + return ".bin" +} diff --git a/internal/infrastructure/docparser/anydoc/backend_stub.go b/internal/infrastructure/docparser/anydoc/backend_stub.go new file mode 100644 index 00000000..7de1b36a --- /dev/null +++ b/internal/infrastructure/docparser/anydoc/backend_stub.go @@ -0,0 +1,23 @@ +//go:build !anydoc || !cgo + +package anydoc + +// The default build links no converter: the anydoc archive is a Rust build +// artifact, and requiring a Rust toolchain to compile WeKnora would be a steep +// price for one optional engine. Everything still compiles and the engine +// registry simply reports the engine as unavailable. + +const unavailableReason = "anydoc engine not built into this binary " + + "(rebuild with `make build-anydoc` / `-tags anydoc`)" + +func backendAvailable() bool { return false } + +func backendUnavailableReason() string { return unavailableReason } + +func backendVersion() string { return "" } + +func backendConvert(_ []byte, _ Options) (*Result, error) { + // Convert checks Available first, so this is only reachable if a caller + // bypasses it. + return nil, ErrUnavailable +} diff --git a/internal/infrastructure/docparser/anydoc/convert_linked_test.go b/internal/infrastructure/docparser/anydoc/convert_linked_test.go new file mode 100644 index 00000000..ba95816d --- /dev/null +++ b/internal/infrastructure/docparser/anydoc/convert_linked_test.go @@ -0,0 +1,333 @@ +//go:build anydoc && cgo + +package anydoc + +import ( + "archive/zip" + "bytes" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// These tests exercise the linked Rust converter, so they only build with the +// `anydoc` tag. Run them with: +// +// scripts/build-anydoc-lib.sh && go test -tags anydoc ./internal/infrastructure/docparser/... + +// SupportedFileTypes is a static list, so that the engine can advertise its +// file types in builds that link no converter. This is the check that it still +// agrees with what the converter actually accepts, in either direction. +func TestSupportedFileTypesMatchTheConverter(t *testing.T) { + for _, fileType := range SupportedFileTypes() { + format, ok := FormatForFile(fileType, "") + if !ok { + t.Errorf("%q is advertised but maps to no format", fileType) + continue + } + if _, err := upstreamFormat(format); err != nil { + t.Errorf("%q maps to format %q, which the converter rejects: %v", fileType, format, err) + } + } +} + +func TestConvertCSV(t *testing.T) { + result, err := Convert([]byte("quarter,widgets\nQ1,12\nQ2,15\n"), Options{Format: "csv"}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if !strings.Contains(result.Markdown, "| quarter | widgets |") { + t.Fatalf("expected a markdown table header, got:\n%s", result.Markdown) + } + if !strings.Contains(result.Markdown, "| Q1 | 12 |") { + t.Fatalf("expected the first data row, got:\n%s", result.Markdown) + } +} + +func TestConvertDocxWithEmbeddedImage(t *testing.T) { + document := buildDocx(t) + + result, err := Convert(document, Options{Format: "docx", WithAssets: true}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if !strings.Contains(result.Markdown, "# Quarterly report") { + t.Fatalf("expected the heading, got:\n%s", result.Markdown) + } + if !strings.Contains(result.Markdown, "Widgets shipped on time.") { + t.Fatalf("expected the body paragraph, got:\n%s", result.Markdown) + } + if !strings.Contains(result.Markdown, "Closing remarks.") { + t.Fatalf("expected the trailing paragraph, got:\n%s", result.Markdown) + } + if len(result.Assets) != 1 { + t.Fatalf("got %d assets, want 1", len(result.Assets)) + } + asset := result.Assets[0] + if asset.Name != "image-1.png" { + t.Errorf("asset name = %q, want image-1.png", asset.Name) + } + if !bytes.Equal(asset.Data, onePixelPNG()) { + t.Errorf("asset data does not round-trip the embedded image") + } + if asset.Alt != "Shipping chart" { + t.Errorf("asset alt = %q, want %q", asset.Alt, "Shipping chart") + } + if asset.Section != "Quarterly report" { + t.Errorf("asset section = %q, want %q", asset.Section, "Quarterly report") + } + imageLink := "![Shipping chart](images/image-1.png)" + if !strings.Contains(result.Markdown, imageLink) { + t.Fatalf("expected the image in place, got:\n%s", result.Markdown) + } + before := strings.Index(result.Markdown, "Widgets shipped on time.") + at := strings.Index(result.Markdown, imageLink) + after := strings.Index(result.Markdown, "Closing remarks.") + if before < 0 || at < 0 || after < 0 || !(before < at && at < after) { + t.Fatalf("image is not between the surrounding paragraphs:\n%s", result.Markdown) + } +} + +func TestConvertTextlessPDFNeedsOCR(t *testing.T) { + _, err := Convert(textlessPDF(), Options{Format: "pdf"}) + if err == nil { + t.Fatal("Convert succeeded on a PDF with no text, want an error") + } + if !PDFNeedsOCR(err) { + t.Fatalf("Convert error = %v, want an OCR-required error", err) + } +} + +// The ABI reports error messages through a thread-local slot that a second +// call reads, so a goroutine resumed on another OS thread would report an +// empty or someone else's message. The binding pins the goroutine for the +// pair; this is the regression test for that. +func TestErrorDetailSurvivesConcurrency(t *testing.T) { + const conversions = 4000 + + var wg sync.WaitGroup + var empty, crossed atomic.Int64 + for i := 0; i < conversions; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + // docx and xlsx failures carry distinctive messages, so a detail + // naming the other format proves it came from the wrong thread. + format, want, other := "docx", "not a readable zip archive", "unreadable workbook" + if i%2 == 1 { + format, want, other = "xlsx", "unreadable workbook", "not a readable zip archive" + } + _, err := Convert([]byte(fmt.Sprintf("garbage %d", i)), Options{Format: format}) + if err == nil { + t.Errorf("Convert(garbage) succeeded") + return + } + detail := err.Error() + switch { + case strings.Contains(detail, other): + crossed.Add(1) + case !strings.Contains(detail, want): + empty.Add(1) + } + }(i) + } + wg.Wait() + + if empty.Load() > 0 || crossed.Load() > 0 { + t.Fatalf("error details lost or crossed between goroutines: lost=%d crossed=%d (of %d)", + empty.Load(), crossed.Load(), conversions) + } +} + +// Detection reads the container itself, so a document whose format is not +// named still converts. +func TestConvertDetectsFormatFromContent(t *testing.T) { + result, err := Convert(buildDocx(t), Options{}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if !strings.Contains(result.Markdown, "# Quarterly report") { + t.Fatalf("expected the heading, got:\n%s", result.Markdown) + } +} + +func TestConvertRejectsGarbage(t *testing.T) { + if _, err := Convert([]byte("not a document at all"), Options{Format: "docx"}); err == nil { + t.Fatal("Convert succeeded on garbage input, want an error") + } +} + +// PDF has no document model, so WithAssets has to be dropped rather than turn +// a working conversion into an error. +func TestConvertPDFIgnoresAssetRequest(t *testing.T) { + result, err := Convert(minimalPDF(), Options{Format: "pdf", WithAssets: true}) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if !strings.Contains(result.Markdown, "Shipping summary") { + t.Fatalf("expected the page text, got:\n%s", result.Markdown) + } + if len(result.Assets) != 0 { + t.Errorf("got %d assets for a PDF, want 0", len(result.Assets)) + } +} + +// A PDF whose catalog holds a deeply nested array is the shape of +// RUSTSEC-2026-0187: lopdf 0.41 recursed per nesting level and killed the +// process with an uncatchable stack overflow, which no Go-side or Rust-side +// recover can contain. The pinned dependency in third_party/anydoc-go bounds +// the recursion, so this must come back as an ordinary error — and if a +// dependency bump ever reintroduces it, this test crashes rather than passing +// quietly. +func TestDeeplyNestedPDFFailsWithoutKillingTheProcess(t *testing.T) { + const depth = 50000 + nested := strings.Repeat("[", depth) + strings.Repeat("]", depth) + + _, err := Convert(nestedPDF(nested), Options{Format: "pdf"}) + if err == nil { + t.Fatal("Convert succeeded on a PDF with no text, want an error") + } +} + +// buildDocx writes the smallest OOXML package that carries a heading, a +// paragraph, and one embedded image. +func buildDocx(t *testing.T) []byte { + t.Helper() + + parts := map[string]string{ + "[Content_Types].xml": ` + + + + + + +`, + "_rels/.rels": ` + + +`, + "word/styles.xml": ` + + + + + +`, + "word/_rels/document.xml.rels": ` + + + +`, + "word/document.xml": ` + + + Quarterly report + Widgets shipped on time. + + + + + + + + Closing remarks. + +`, + } + + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + for name, content := range parts { + writer, err := archive.Create(name) + if err != nil { + t.Fatalf("create %s: %v", name, err) + } + if _, err := writer.Write([]byte(content)); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + image, err := archive.Create("word/media/image1.png") + if err != nil { + t.Fatalf("create image part: %v", err) + } + if _, err := image.Write(onePixelPNG()); err != nil { + t.Fatalf("write image part: %v", err) + } + if err := archive.Close(); err != nil { + t.Fatalf("close archive: %v", err) + } + return buf.Bytes() +} + +// onePixelPNG is a 1x1 transparent PNG: the smallest thing a container will +// accept as an image part. +func onePixelPNG() []byte { + return []byte{ + 0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, + 0x00, 0x00, 0x00, 0x0d, 'I', 'H', 'D', 'R', + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, + 0x00, 0x00, 0x00, 0x0a, 'I', 'D', 'A', 'T', + 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, 0x05, 0x00, 0x01, + 0x0d, 0x0a, 0x2d, 0xb4, + 0x00, 0x00, 0x00, 0x00, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82, + } +} + +// minimalPDF is a one-page PDF with a single text run, written by hand so the +// test carries no binary fixture. +func minimalPDF() []byte { + content := "BT /F1 12 Tf 20 100 Td (Shipping summary) Tj ET\n" + return writePDF([]string{ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R " + + "/Resources << /Font << /F1 5 0 R >> >> >>", + fmt.Sprintf("<< /Length %d >>\nstream\n%sendstream", len(content), content), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + }) +} + +// nestedPDF is a one-page PDF carrying the given nested value in its catalog. +func nestedPDF(nested string) []byte { + return writePDF([]string{ + "<< /Type /Catalog /Pages 2 0 R /Nested " + nested + " >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + }) +} + +// textlessPDF is a one-page PDF with no text stream: the shape of a scanned +// exam paper as far as pdf-inspector is concerned. +func textlessPDF() []byte { + return writePDF([]string{ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + }) +} + +func writePDF(objects []string) []byte { + var pdf bytes.Buffer + offsets := make([]int, len(objects)) + pdf.WriteString("%PDF-1.4\n") + for i, object := range objects { + offsets[i] = pdf.Len() + fmt.Fprintf(&pdf, "%d 0 obj\n%s\nendobj\n", i+1, object) + } + + xref := pdf.Len() + fmt.Fprintf(&pdf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1) + for _, offset := range offsets { + fmt.Fprintf(&pdf, "%010d 00000 n \n", offset) + } + fmt.Fprintf(&pdf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref) + return pdf.Bytes() +} diff --git a/internal/infrastructure/docparser/anydoc_reader.go b/internal/infrastructure/docparser/anydoc_reader.go new file mode 100644 index 00000000..10a38e47 --- /dev/null +++ b/internal/infrastructure/docparser/anydoc_reader.go @@ -0,0 +1,159 @@ +package docparser + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// AnydocReader converts office documents to markdown in this process, using +// the anydoc converter instead of the Python docreader service. +// +// Embedded images are rendered in place from the document model. PDFs with no +// text layer (scanned exam papers) cannot be converted by anydoc; when a +// fallback reader is configured (typically the builtin docreader) those files +// are handed off so pages can be rasterized for OCR. +type AnydocReader struct { + // extractImages controls whether embedded images are parsed and placed + // in the markdown. Off halves parse time for text-only indexing. + extractImages bool + // fallback is used when a PDF has no extractable text. Nil means the + // conversion error is returned to the caller. + fallback interfaces.DocReader +} + +// NewAnydocReader builds a reader. The "anydoc_extract_images" override turns +// off image extraction. fallback may be nil. +func NewAnydocReader(overrides map[string]string, fallback interfaces.DocReader) *AnydocReader { + return &AnydocReader{ + extractImages: !isFalsey(overrides["anydoc_extract_images"]), + fallback: fallback, + } +} + +// Read converts the document carried by the request. +func (r *AnydocReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) { + if req.URL != "" && len(req.FileContent) == 0 { + return nil, fmt.Errorf("anydoc engine reads uploaded documents, not URLs") + } + + format, ok := anydoc.FormatForFile(req.FileType, req.FileName) + if !ok { + return nil, fmt.Errorf("anydoc engine does not support file type %q", fileTypeOf(req)) + } + + converted, err := anydoc.Convert(req.FileContent, anydoc.Options{ + Format: format, + WithAssets: r.extractImages, + }) + if format == "pdf" && (anydoc.PDFNeedsOCR(err) || pdfMarkdownIsEmpty(converted, err)) { + return r.readScannedPDF(ctx, req, err) + } + if err != nil { + return nil, fmt.Errorf("anydoc conversion failed for %q: %w", req.FileName, err) + } + if converted.AssetsError != nil { + logger.Warnf(ctx, "[anydoc] %q parsed as text but its images were dropped: %v", + req.FileName, converted.AssetsError) + } + + return &types.ReadResult{ + MarkdownContent: converted.Markdown, + ImageRefs: imageRefsFromAssets(converted.Assets), + Metadata: map[string]string{ + "parser": AnydocEngineName, + "anydoc_version": anydoc.Version(), + "source_format": format, + }, + }, nil +} + +func pdfMarkdownIsEmpty(converted *anydoc.Result, err error) bool { + if err != nil || converted == nil { + return false + } + return strings.TrimSpace(converted.Markdown) == "" +} + +func (r *AnydocReader) readScannedPDF(ctx context.Context, req *types.ReadRequest, convertErr error) (*types.ReadResult, error) { + if !r.canFallback() { + if convertErr != nil { + return nil, fmt.Errorf("anydoc conversion failed for %q: %w", req.FileName, convertErr) + } + return nil, fmt.Errorf("anydoc conversion failed for %q: PDF has no extractable text; OCR is required", req.FileName) + } + + logger.Infof(ctx, "[anydoc] %q has no text layer, falling back to builtin for scanned-PDF OCR", req.FileName) + fallbackReq := *req + fallbackReq.ParserEngine = BuiltinEngineName + result, err := r.fallback.Read(ctx, &fallbackReq) + if err != nil { + return nil, fmt.Errorf("anydoc scanned-PDF fallback failed for %q: %w", req.FileName, err) + } + if result == nil { + return nil, fmt.Errorf("anydoc scanned-PDF fallback returned no result for %q", req.FileName) + } + if result.Metadata == nil { + result.Metadata = map[string]string{} + } + if result.Metadata["parser"] == "" { + result.Metadata["parser"] = BuiltinEngineName + } + result.Metadata["anydoc_fallback"] = "scanned_pdf" + if result.Metadata["image_source_type"] == "" { + result.Metadata["image_source_type"] = "scanned_pdf" + } + return result, nil +} + +func (r *AnydocReader) canFallback() bool { + if r.fallback == nil { + return false + } + if connected, ok := r.fallback.(interface{ IsConnected() bool }); ok { + return connected.IsConnected() + } + return true +} + +// imageRefsFromAssets turns extracted assets into ImageRefs that match the +// in-place markdown links renderMarkdown wrote. +func imageRefsFromAssets(assets []anydoc.Asset) []types.ImageRef { + if len(assets) == 0 { + return nil + } + refs := make([]types.ImageRef, 0, len(assets)) + for _, asset := range assets { + ref := anydoc.ImageDir + asset.Name + refs = append(refs, types.ImageRef{ + Filename: asset.Name, + OriginalRef: ref, + MimeType: asset.MediaType, + ImageData: asset.Data, + }) + } + return refs +} + +// fileTypeOf reports the request's file type, falling back to the file name's +// extension the way every reader in this package does. +func fileTypeOf(req *types.ReadRequest) string { + if ft := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(req.FileType)), "."); ft != "" { + return ft + } + return strings.TrimPrefix(strings.ToLower(filepath.Ext(req.FileName)), ".") +} + +func isFalsey(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "false", "0", "no", "off": + return true + } + return false +} diff --git a/internal/infrastructure/docparser/anydoc_reader_test.go b/internal/infrastructure/docparser/anydoc_reader_test.go new file mode 100644 index 00000000..d09bf4ae --- /dev/null +++ b/internal/infrastructure/docparser/anydoc_reader_test.go @@ -0,0 +1,144 @@ +package docparser + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestAnydocReaderRejectsUnsupportedFileType(t *testing.T) { + _, err := NewAnydocReader(nil, nil).Read(context.Background(), &types.ReadRequest{ + FileName: "photo.png", + FileType: "png", + FileContent: []byte("not really a png"), + }) + if err == nil { + t.Fatal("Read succeeded for an unsupported file type, want an error") + } + if !strings.Contains(err.Error(), "png") { + t.Errorf("error does not name the file type: %v", err) + } +} + +func TestAnydocReaderRejectsURLs(t *testing.T) { + _, err := NewAnydocReader(nil, nil).Read(context.Background(), &types.ReadRequest{ + URL: "https://example.com/report.docx", + FileType: "docx", + }) + if err == nil { + t.Fatal("Read succeeded for a URL request, want an error") + } +} + +func TestNewAnydocReaderHonoursImageExtractionOverride(t *testing.T) { + if !NewAnydocReader(nil, nil).extractImages { + t.Error("image extraction is off by default, want on") + } + if NewAnydocReader(map[string]string{"anydoc_extract_images": "false"}, nil).extractImages { + t.Error("image extraction stayed on after the override turned it off") + } +} + +func TestImageRefsFromAssetsMatchInPlaceMarkdownPaths(t *testing.T) { + refs := imageRefsFromAssets([]anydoc.Asset{ + {Name: "image-1.png", MediaType: "image/png", Data: []byte("first")}, + {Name: "image-2.jpg", MediaType: "image/jpeg", Data: []byte("second")}, + }) + if len(refs) != 2 { + t.Fatalf("got %d image refs, want 2", len(refs)) + } + if refs[0].OriginalRef != "images/image-1.png" { + t.Errorf("first OriginalRef = %q, want images/image-1.png", refs[0].OriginalRef) + } + if refs[1].MimeType != "image/jpeg" { + t.Errorf("second mime type = %q, want image/jpeg", refs[1].MimeType) + } + if len(refs[0].ImageData) == 0 { + t.Error("image ref carries no bytes") + } +} + +func TestImageRefsFromAssetsEmpty(t *testing.T) { + if refs := imageRefsFromAssets(nil); refs != nil { + t.Errorf("got %d image refs, want none", len(refs)) + } +} + +func TestAnydocReaderFallsBackForScannedPDF(t *testing.T) { + fallback := &stubDocReader{result: &types.ReadResult{ + MarkdownContent: "![page](images/page_1.jpg)", + ImageRefs: []types.ImageRef{{ + Filename: "page_1.jpg", + OriginalRef: "images/page_1.jpg", + ImageData: []byte("jpeg"), + }}, + Metadata: map[string]string{"image_source_type": "scanned_pdf"}, + }} + result, err := NewAnydocReader(nil, fallback).readScannedPDF(context.Background(), &types.ReadRequest{ + FileName: "exam.pdf", + FileType: "pdf", + }, errors.New("PDF has no extractable text (Scanned, 5 pages): OCR is required")) + if err != nil { + t.Fatalf("readScannedPDF: %v", err) + } + if !strings.Contains(result.MarkdownContent, "images/page_1.jpg") { + t.Errorf("fallback markdown missing page image:\n%s", result.MarkdownContent) + } + if result.Metadata["anydoc_fallback"] != "scanned_pdf" { + t.Errorf("anydoc_fallback = %q, want scanned_pdf", result.Metadata["anydoc_fallback"]) + } + if result.Metadata["image_source_type"] != "scanned_pdf" { + t.Errorf("image_source_type = %q, want scanned_pdf", result.Metadata["image_source_type"]) + } + if fallback.got == nil || fallback.got.ParserEngine != BuiltinEngineName { + t.Errorf("fallback request engine = %v, want builtin", fallback.got) + } +} + +func TestAnydocReaderScannedPDFWithoutFallbackKeepsError(t *testing.T) { + _, err := NewAnydocReader(nil, nil).readScannedPDF(context.Background(), &types.ReadRequest{ + FileName: "exam.pdf", + }, errors.New("PDF has no extractable text: OCR is required")) + if err == nil { + t.Fatal("readScannedPDF succeeded without a fallback, want an error") + } + if !strings.Contains(err.Error(), "OCR is required") { + t.Errorf("error does not mention OCR: %v", err) + } +} + +func TestAnydocReaderDisconnectedFallbackKeepsError(t *testing.T) { + fallback := &disconnectedDocReader{} + _, err := NewAnydocReader(nil, fallback).readScannedPDF(context.Background(), &types.ReadRequest{ + FileName: "exam.pdf", + }, errors.New("PDF has no extractable text: OCR is required")) + if err == nil { + t.Fatal("readScannedPDF succeeded with a disconnected fallback, want an error") + } + if !strings.Contains(err.Error(), "OCR is required") { + t.Errorf("error does not mention OCR: %v", err) + } +} + +type stubDocReader struct { + got *types.ReadRequest + result *types.ReadResult + err error +} + +func (s *stubDocReader) Read(_ context.Context, req *types.ReadRequest) (*types.ReadResult, error) { + s.got = req + return s.result, s.err +} + +type disconnectedDocReader struct{} + +func (disconnectedDocReader) Read(context.Context, *types.ReadRequest) (*types.ReadResult, error) { + return nil, errors.New("should not be called") +} + +func (disconnectedDocReader) IsConnected() bool { return false } diff --git a/internal/infrastructure/docparser/doc.go b/internal/infrastructure/docparser/doc.go new file mode 100644 index 00000000..d3bc5cf8 --- /dev/null +++ b/internal/infrastructure/docparser/doc.go @@ -0,0 +1,21 @@ +// Package docparser turns uploaded files and URLs into Markdown plus the +// images they contain, which is the input the chunking and multimodal stages +// downstream expect. +// +// A parse request names a parser engine, and every engine is one entry in the +// registry: +// +// - engines.go declares the catalog — each engine's name, description, +// supported file types, availability probe, and the reader that does the +// work. +// - engine_registry.go turns an engine name into a reader (NewReader) and +// merges the local catalog with the engines the Python docreader reports +// over its ListEngines RPC (ListAllEngines). +// +// Readers differ in where the parsing happens: in this process +// (SimpleFormatReader, AnydocReader), in the docreader service over gRPC or +// HTTP (GRPCDocumentReader, HTTPDocumentReader), or in a remote API (MinerU, +// PaddleOCR-VL, WeKnora Cloud). They all return types.ReadResult, so the rest +// of the package — image resolution and storage, table normalization — is +// shared regardless of which engine ran. +package docparser diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 4d59f966..99879218 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -1,182 +1,92 @@ package docparser import ( - "strings" + "context" + "fmt" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" ) -// EngineRegistration is the interface every locally registered parser engine -// must implement. Remote-only engines (e.g. markitdown) are discovered via -// the docreader ListEngines RPC and do not need a local registration. +// EngineRegistration is what every locally registered parser engine provides: +// the metadata the engine list shows, and the reader that does the parsing. +// Remote-only engines (e.g. markitdown) live in the Python docreader, are +// discovered through its ListEngines RPC, and never register here — the +// registry routes them to the docreader client by default. type EngineRegistration interface { Name() string Description() string FileTypes(docreaderConnected bool) []string CheckAvailable(docreaderConnected bool, overrides map[string]string) (available bool, reason string) + // NewReader builds the reader for one parse request. Returning an error + // means this engine cannot serve the request (missing credentials, + // unreachable service); the caller reports it rather than silently + // parsing with something else. + NewReader(ctx context.Context, deps ReaderDeps) (interfaces.DocReader, error) } -// localEngines holds all locally registered parser engines. +// ReaderDeps carries everything an engine may need to build its reader but +// cannot construct itself: tenant configuration, tenant credentials, and the +// shared docreader connection. +type ReaderDeps struct { + // Overrides holds tenant-level engine configuration (service endpoints, + // API keys), as produced by ParserEngineConfig.ToOverridesMap. + Overrides map[string]string + // Remote is the docreader client. Nil when the service is not connected. + Remote interfaces.DocReader + // WeKnoraCloudCredentials resolves the tenant's WeKnora Cloud + // credentials. It is a function rather than a value because resolving + // them can hit the database, which most engines never need. Nil, or a + // nil return, means the tenant has not configured them. + WeKnoraCloudCredentials func(ctx context.Context) *types.WeKnoraCloudCredentials +} + +// localEngines holds all locally registered parser engines, in registration +// order — which is also the order the engine list is shown in. var localEngines []EngineRegistration -// RegisterEngine adds an engine to the local registry. Called in init(). +// RegisterEngine adds an engine to the local registry. Called from init(). func RegisterEngine(e EngineRegistration) { localEngines = append(localEngines, e) } -func init() { - RegisterEngine(&builtinEngine{}) - RegisterEngine(&simpleEngine{}) - RegisterEngine(&weKnoraCloudEngine{}) - RegisterEngine(&mineruEngine{}) - RegisterEngine(&mineruCloudEngine{}) - RegisterEngine(&paddleOCRVLEngine{}) - RegisterEngine(&paddleOCRVLCloudEngine{}) -} - -// --------------------------------------------------------------------------- -// builtin — DocReader-backed parser for complex document formats. -// --------------------------------------------------------------------------- - -type builtinEngine struct{} - -func (e *builtinEngine) Name() string { return "builtin" } -func (e *builtinEngine) Description() string { - return "DocReader built-in parser engine" -} -func (e *builtinEngine) FileTypes(_ bool) []string { - return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "epub", "html", "htm", "mhtml", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} -} -func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) { - if docreaderConnected { - return true, "" +// lookupEngine returns the locally registered engine with this name. +func lookupEngine(name string) (EngineRegistration, bool) { + for _, engine := range localEngines { + if engine.Name() == name { + return engine, true + } } - return false, "DocReader service not connected" + return nil, false } -// SimpleEngineName is the engine name for Go-native simple format handling. -const SimpleEngineName = "simple" - -// WeKnoraCloudEngineName is the engine name for WeKnoraCloud-backed document parsing. -const WeKnoraCloudEngineName = "weknoracloud" - -// --------------------------------------------------------------------------- -// simple — Go handles md/txt/csv natively, no external service needed. -// Distinct from docreader's "builtin" which uses Python libraries for -// complex formats (docx, pdf, etc.). -// --------------------------------------------------------------------------- - -type simpleEngine struct{} - -func (e *simpleEngine) Name() string { return SimpleEngineName } -func (e *simpleEngine) Description() string { - return "Simple format & image parsing (no external service required)" -} -func (e *simpleEngine) FileTypes(_ bool) []string { - return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "mp3", "wav", "m4a", "flac", "ogg"} -} -func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { - return true, "" -} - -// --------------------------------------------------------------------------- -// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests. -// --------------------------------------------------------------------------- - -type weKnoraCloudEngine struct{} - -func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName } -func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" } -func (e *weKnoraCloudEngine) FileTypes(_ bool) []string { - return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"} -} -func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) { - if overrides["weknoracloud_app_id"] != "" { - return true, "" +// NewReader builds the reader for an engine. +// +// An empty engine name means "no explicit choice": simple formats are handled +// in Go and everything else goes to the docreader service. An unknown name is +// routed to the docreader too, so engines that only exist in the Python +// service keep working without a Go-side registration. +func NewReader( + ctx context.Context, engine, fileType string, isURL bool, deps ReaderDeps, +) (interfaces.DocReader, error) { + if registration, ok := lookupEngine(engine); ok { + return registration.NewReader(ctx, deps) } - return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up." -} - -// --------------------------------------------------------------------------- -// mineru — Go-native, calls self-hosted MinerU API directly -// --------------------------------------------------------------------------- - -type mineruEngine struct{} - -func (e *mineruEngine) Name() string { return "mineru" } -func (e *mineruEngine) Description() string { return "MinerU self-hosted service" } -func (e *mineruEngine) FileTypes(_ bool) []string { - return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"} -} -func (e *mineruEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { - endpoint := strings.TrimSpace(overrides["mineru_endpoint"]) - if endpoint == "" { - return false, "MinerU service not configured" + if engine == "" && !isURL && IsSimpleFormat(fileType) { + return &SimpleFormatReader{}, nil } - return PingMinerU(endpoint) + return remoteReader(deps) } -// --------------------------------------------------------------------------- -// mineru_cloud — Go-native, calls MinerU Cloud API directly -// --------------------------------------------------------------------------- - -type mineruCloudEngine struct{} - -func (e *mineruCloudEngine) Name() string { return "mineru_cloud" } -func (e *mineruCloudEngine) Description() string { return "MinerU Cloud API" } -func (e *mineruCloudEngine) FileTypes(_ bool) []string { - return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"} -} -func (e *mineruCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { - apiKey := strings.TrimSpace(overrides["mineru_api_key"]) - if apiKey == "" { - return false, "MinerU API Key not configured" +// remoteReader returns the docreader client, or an error when the service is +// not connected — a nil interface value here would panic at the call site. +func remoteReader(deps ReaderDeps) (interfaces.DocReader, error) { + if deps.Remote == nil { + return nil, errNotConnected } - return PingMinerUCloud(apiKey) + return deps.Remote, nil } -// --------------------------------------------------------------------------- -// paddleocr_vl — Go-native, calls a self-hosted PaddleOCR-VL pipeline service -// --------------------------------------------------------------------------- - -type paddleOCRVLEngine struct{} - -func (e *paddleOCRVLEngine) Name() string { return "paddleocr_vl" } -func (e *paddleOCRVLEngine) Description() string { return "PaddleOCR-VL self-hosted service" } -func (e *paddleOCRVLEngine) FileTypes(_ bool) []string { - return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"} -} -func (e *paddleOCRVLEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { - endpoint := strings.TrimSpace(overrides["paddleocr_vl_endpoint"]) - if endpoint == "" { - return false, "PaddleOCR-VL service not configured" - } - return PingPaddleOCRVL(endpoint) -} - -// --------------------------------------------------------------------------- -// paddleocr_vl_cloud — Go-native, calls the PaddleOCR-VL AI Studio cloud API -// --------------------------------------------------------------------------- - -type paddleOCRVLCloudEngine struct{} - -func (e *paddleOCRVLCloudEngine) Name() string { return "paddleocr_vl_cloud" } -func (e *paddleOCRVLCloudEngine) Description() string { return "PaddleOCR-VL Cloud API" } -func (e *paddleOCRVLCloudEngine) FileTypes(_ bool) []string { - return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"} -} -func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { - token := strings.TrimSpace(overrides["paddleocr_vl_cloud_token"]) - if token == "" { - return false, "PaddleOCR-VL Cloud Token not configured" - } - return PingPaddleOCRVLCloud(token) -} - -// --------------------------------------------------------------------------- -// ListAllEngines — merge local + remote -// --------------------------------------------------------------------------- - // ListAllEngines returns the merged engine list: locally registered engines // plus engines discovered from the remote docreader via ListEngines RPC. // @@ -187,7 +97,9 @@ func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]str // authoritative for its own capabilities). // - Remote engines not present locally are appended as-is, enabling // auto-discovery of newly added docreader engines without Go changes. -func ListAllEngines(docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo) []types.ParserEngineInfo { +func ListAllEngines( + docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo, +) []types.ParserEngineInfo { remoteMap := make(map[string]types.ParserEngineInfo, len(remoteEngines)) for _, re := range remoteEngines { remoteMap[re.Name] = re @@ -231,3 +143,9 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote return result } + +// errEngineUnavailable reports an engine that is registered but cannot run for +// this tenant or this build. +func errEngineUnavailable(engine, reason string) error { + return fmt.Errorf("parser engine %q is unavailable: %s", engine, reason) +} diff --git a/internal/infrastructure/docparser/engine_registry_reader_test.go b/internal/infrastructure/docparser/engine_registry_reader_test.go new file mode 100644 index 00000000..288562ab --- /dev/null +++ b/internal/infrastructure/docparser/engine_registry_reader_test.go @@ -0,0 +1,111 @@ +package docparser + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// stubRemote stands in for the docreader client. NewReader only ever hands it +// back, so the embedded interface is never called — and would panic if routing +// ever sent a parse here by mistake. +type stubRemote struct{ interfaces.DocReader } + +func TestNewReaderRoutesByEngine(t *testing.T) { + ctx := context.Background() + remote := &stubRemote{} + deps := ReaderDeps{Remote: remote} + + cases := []struct { + name string + engine string + fileType string + isURL bool + want any + }{ + {name: "simple engine", engine: SimpleEngineName, fileType: "md", want: &SimpleFormatReader{}}, + {name: "builtin engine goes remote", engine: BuiltinEngineName, fileType: "md", want: remote}, + {name: "unset engine handles simple formats in Go", fileType: "csv", want: &SimpleFormatReader{}}, + {name: "unset engine sends complex formats to docreader", fileType: "docx", want: remote}, + {name: "URLs always go to docreader", fileType: "md", isURL: true, want: remote}, + {name: "docreader-only engines fall through", engine: "markitdown", fileType: "docx", want: remote}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reader, err := NewReader(ctx, tc.engine, tc.fileType, tc.isURL, deps) + if err != nil { + t.Fatalf("NewReader: %v", err) + } + if _, isSimple := tc.want.(*SimpleFormatReader); isSimple { + if _, ok := reader.(*SimpleFormatReader); !ok { + t.Fatalf("reader = %T, want *SimpleFormatReader", reader) + } + return + } + if reader != tc.want { + t.Fatalf("reader = %T, want the docreader client", reader) + } + }) + } +} + +// A disconnected docreader has to surface as an error, not as a nil reader the +// caller would dereference. +func TestNewReaderReportsDisconnectedDocReader(t *testing.T) { + if _, err := NewReader(context.Background(), BuiltinEngineName, "docx", false, ReaderDeps{}); err == nil { + t.Fatal("NewReader succeeded without a docreader connection, want an error") + } +} + +func TestNewReaderRequiresWeKnoraCloudCredentials(t *testing.T) { + _, err := NewReader(context.Background(), WeKnoraCloudEngineName, "docx", false, ReaderDeps{ + WeKnoraCloudCredentials: func(context.Context) *types.WeKnoraCloudCredentials { return nil }, + }) + if err == nil { + t.Fatal("NewReader succeeded without credentials, want an error") + } +} + +// The anydoc engine is only linked into builds tagged `anydoc`; everywhere +// else it must be listed as unavailable and refuse to build a reader, so a +// knowledge base configured for it fails loudly instead of parsing with +// something else. +func TestAnydocEngineFollowsBuildAvailability(t *testing.T) { + reader, err := NewReader(context.Background(), AnydocEngineName, "docx", false, ReaderDeps{}) + + if anydoc.Available() { + if err != nil { + t.Fatalf("NewReader: %v", err) + } + if _, ok := reader.(*AnydocReader); !ok { + t.Fatalf("reader = %T, want *AnydocReader", reader) + } + return + } + if err == nil { + t.Fatal("NewReader succeeded without the converter linked in, want an error") + } +} + +func TestListAllEnginesIncludesAnydoc(t *testing.T) { + for _, engine := range ListAllEngines(true, nil, nil) { + if engine.Name != AnydocEngineName { + continue + } + if engine.Available != anydoc.Available() { + t.Errorf("anydoc availability = %v, want %v", engine.Available, anydoc.Available()) + } + if !engine.Available && engine.UnavailableReason == "" { + t.Error("anydoc is unavailable without a reason to show") + } + if len(engine.FileTypes) == 0 { + t.Error("anydoc lists no file types") + } + return + } + t.Fatal("anydoc engine not found in the engine list") +} diff --git a/internal/infrastructure/docparser/engines.go b/internal/infrastructure/docparser/engines.go new file mode 100644 index 00000000..f45c9e95 --- /dev/null +++ b/internal/infrastructure/docparser/engines.go @@ -0,0 +1,271 @@ +package docparser + +import ( + "context" + "strings" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// Engine names, as stored in knowledge base parser rules and shown in the UI. +const ( + // BuiltinEngineName is the DocReader (Python) parser suite. + BuiltinEngineName = "builtin" + // SimpleEngineName is Go-native handling of text formats and images. + SimpleEngineName = "simple" + // AnydocEngineName is the in-process anydoc office-document converter. + AnydocEngineName = "anydoc" + // WeKnoraCloudEngineName is the hosted WeKnora Cloud document reader. + WeKnoraCloudEngineName = "weknoracloud" + // MinerUEngineName is a self-hosted MinerU service. + MinerUEngineName = "mineru" + // MinerUCloudEngineName is the MinerU Cloud API. + MinerUCloudEngineName = "mineru_cloud" + // PaddleOCRVLEngineName is a self-hosted PaddleOCR-VL pipeline. + PaddleOCRVLEngineName = "paddleocr_vl" + // PaddleOCRVLCloudEngineName is the PaddleOCR-VL AI Studio cloud API. + PaddleOCRVLCloudEngineName = "paddleocr_vl_cloud" +) + +func init() { + RegisterEngine(&builtinEngine{}) + RegisterEngine(&simpleEngine{}) + RegisterEngine(&anydocEngine{}) + RegisterEngine(&weKnoraCloudEngine{}) + RegisterEngine(&mineruEngine{}) + RegisterEngine(&mineruCloudEngine{}) + RegisterEngine(&paddleOCRVLEngine{}) + RegisterEngine(&paddleOCRVLCloudEngine{}) +} + +// --------------------------------------------------------------------------- +// builtin — DocReader-backed parser for complex document formats. +// --------------------------------------------------------------------------- + +type builtinEngine struct{} + +func (e *builtinEngine) Name() string { return BuiltinEngineName } + +func (e *builtinEngine) Description() string { return "DocReader built-in parser engine" } + +func (e *builtinEngine) FileTypes(_ bool) []string { + return []string{ + "docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "epub", + "html", "htm", "mhtml", + "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", + "mp3", "wav", "m4a", "flac", "ogg", + } +} + +func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]string) (bool, string) { + if docreaderConnected { + return true, "" + } + return false, "DocReader service not connected" +} + +// NewReader returns the docreader client. Selecting "builtin" explicitly means +// the docreader is wanted even for formats the simple reader could handle. +func (e *builtinEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) { + return remoteReader(deps) +} + +// --------------------------------------------------------------------------- +// simple — Go handles md/txt/csv/json natively, no external service needed. +// Distinct from docreader's "builtin", which uses Python libraries for complex +// formats (docx, pdf, ...). +// --------------------------------------------------------------------------- + +type simpleEngine struct{} + +func (e *simpleEngine) Name() string { return SimpleEngineName } + +func (e *simpleEngine) Description() string { + return "Simple format & image parsing (no external service required)" +} + +func (e *simpleEngine) FileTypes(_ bool) []string { + return []string{ + "md", "markdown", "txt", "csv", "json", + "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", + "mp3", "wav", "m4a", "flac", "ogg", + } +} + +func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { + return true, "" +} + +func (e *simpleEngine) NewReader(_ context.Context, _ ReaderDeps) (interfaces.DocReader, error) { + return &SimpleFormatReader{}, nil +} + +// --------------------------------------------------------------------------- +// anydoc — office documents converted in this process, no external service. +// Only present in builds that link the converter (see the anydoc package). +// --------------------------------------------------------------------------- + +type anydocEngine struct{} + +func (e *anydocEngine) Name() string { return AnydocEngineName } + +func (e *anydocEngine) Description() string { + return "anydoc in-process office document converter (no external service required)" +} + +func (e *anydocEngine) FileTypes(_ bool) []string { return anydoc.SupportedFileTypes() } + +func (e *anydocEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { + if anydoc.Available() { + return true, "" + } + return false, anydoc.UnavailableReason() +} + +func (e *anydocEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) { + if !anydoc.Available() { + return nil, errEngineUnavailable(AnydocEngineName, anydoc.UnavailableReason()) + } + return NewAnydocReader(deps.Overrides, deps.Remote), nil +} + +// --------------------------------------------------------------------------- +// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests. +// --------------------------------------------------------------------------- + +type weKnoraCloudEngine struct{} + +func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName } + +func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" } + +func (e *weKnoraCloudEngine) FileTypes(_ bool) []string { + return []string{"docx", "doc", "pdf", "md", "markdown", "xlsx", "xls", "pptx", "ppt"} +} + +func (e *weKnoraCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { + if overrides["weknoracloud_app_id"] != "" { + return true, "" + } + return false, "WeKnora Cloud credentials not configured. Go to Settings → WeKnora Cloud to set up." +} + +func (e *weKnoraCloudEngine) NewReader( + ctx context.Context, deps ReaderDeps, +) (interfaces.DocReader, error) { + if deps.WeKnoraCloudCredentials == nil { + return nil, errEngineUnavailable(WeKnoraCloudEngineName, "no credential resolver configured") + } + creds := deps.WeKnoraCloudCredentials(ctx) + if creds == nil { + return nil, errEngineUnavailable(WeKnoraCloudEngineName, "tenant credentials not configured") + } + return NewWeKnoraCloudSignedDocumentReader(creds.AppID, creds.AppSecret) +} + +// --------------------------------------------------------------------------- +// mineru — Go-native, calls self-hosted MinerU API directly +// --------------------------------------------------------------------------- + +type mineruEngine struct{} + +func (e *mineruEngine) Name() string { return MinerUEngineName } + +func (e *mineruEngine) Description() string { return "MinerU self-hosted service" } + +func (e *mineruEngine) FileTypes(_ bool) []string { + return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"} +} + +func (e *mineruEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { + endpoint := strings.TrimSpace(overrides["mineru_endpoint"]) + if endpoint == "" { + return false, "MinerU service not configured" + } + return PingMinerU(endpoint) +} + +func (e *mineruEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) { + return NewMinerUReader(deps.Overrides), nil +} + +// --------------------------------------------------------------------------- +// mineru_cloud — Go-native, calls MinerU Cloud API directly +// --------------------------------------------------------------------------- + +type mineruCloudEngine struct{} + +func (e *mineruCloudEngine) Name() string { return MinerUCloudEngineName } + +func (e *mineruCloudEngine) Description() string { return "MinerU Cloud API" } + +func (e *mineruCloudEngine) FileTypes(_ bool) []string { + return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"} +} + +func (e *mineruCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { + apiKey := strings.TrimSpace(overrides["mineru_api_key"]) + if apiKey == "" { + return false, "MinerU API Key not configured" + } + return PingMinerUCloud(apiKey) +} + +func (e *mineruCloudEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) { + return NewMinerUCloudReader(deps.Overrides), nil +} + +// --------------------------------------------------------------------------- +// paddleocr_vl — Go-native, calls a self-hosted PaddleOCR-VL pipeline service +// --------------------------------------------------------------------------- + +type paddleOCRVLEngine struct{} + +func (e *paddleOCRVLEngine) Name() string { return PaddleOCRVLEngineName } + +func (e *paddleOCRVLEngine) Description() string { return "PaddleOCR-VL self-hosted service" } + +func (e *paddleOCRVLEngine) FileTypes(_ bool) []string { + return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"} +} + +func (e *paddleOCRVLEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { + endpoint := strings.TrimSpace(overrides["paddleocr_vl_endpoint"]) + if endpoint == "" { + return false, "PaddleOCR-VL service not configured" + } + return PingPaddleOCRVL(endpoint) +} + +func (e *paddleOCRVLEngine) NewReader(_ context.Context, deps ReaderDeps) (interfaces.DocReader, error) { + return NewPaddleOCRVLReader(deps.Overrides), nil +} + +// --------------------------------------------------------------------------- +// paddleocr_vl_cloud — Go-native, calls the PaddleOCR-VL AI Studio cloud API +// --------------------------------------------------------------------------- + +type paddleOCRVLCloudEngine struct{} + +func (e *paddleOCRVLCloudEngine) Name() string { return PaddleOCRVLCloudEngineName } + +func (e *paddleOCRVLCloudEngine) Description() string { return "PaddleOCR-VL Cloud API" } + +func (e *paddleOCRVLCloudEngine) FileTypes(_ bool) []string { + return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff"} +} + +func (e *paddleOCRVLCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) { + token := strings.TrimSpace(overrides["paddleocr_vl_cloud_token"]) + if token == "" { + return false, "PaddleOCR-VL Cloud Token not configured" + } + return PingPaddleOCRVLCloud(token) +} + +func (e *paddleOCRVLCloudEngine) NewReader( + _ context.Context, deps ReaderDeps, +) (interfaces.DocReader, error) { + return NewPaddleOCRVLCloudReader(deps.Overrides), nil +} diff --git a/scripts/build-anydoc-lib.sh b/scripts/build-anydoc-lib.sh new file mode 100755 index 00000000..ed70478a --- /dev/null +++ b/scripts/build-anydoc-lib.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Build the anydoc static archive that the `anydoc` build tag links. +# +# The archive is a Rust build artifact (~30 MB), so it is neither committed nor +# downloaded: this script builds it from third_party/anydoc-go, which pins the +# published anydoc crate, and drops it where cgo looks for it. +# +# Usage: +# scripts/build-anydoc-lib.sh # host platform +# TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +crate_dir="$repo_root/third_party/anydoc-go" + +if ! command -v cargo >/dev/null 2>&1; then + echo "error: cargo not found. Install Rust (https://rustup.rs) to build the anydoc archive." >&2 + exit 1 +fi + +# The Rust target triple decides the archive; the directory name mirrors what +# the cgo LDFLAGS in third_party/anydoc-go expect for that platform. +target=${TARGET:-$(rustc -vV | sed -n 's/^host: //p')} +case "$target" in + x86_64-apple-darwin) lib_dir=darwin_amd64 ;; + aarch64-apple-darwin) lib_dir=darwin_arm64 ;; + x86_64-pc-windows-msvc) lib_dir=windows_amd64 ;; + x86_64-unknown-linux-gnu) lib_dir=linux_amd64_gnu ;; + aarch64-unknown-linux-gnu) lib_dir=linux_arm64_gnu ;; + x86_64-unknown-linux-musl) lib_dir=linux_amd64_musl ;; + aarch64-unknown-linux-musl) lib_dir=linux_arm64_musl ;; + *) + echo "error: unsupported target '$target'." >&2 + echo "Supported: {x86_64,aarch64}-{apple-darwin,unknown-linux-{gnu,musl}}, x86_64-pc-windows-msvc" >&2 + exit 1 + ;; +esac + +case "$target" in + *windows-msvc) lib_name=anydoc_go.lib ;; + *) lib_name=libanydoc_go.a ;; +esac + +# document_to_markdown is private in published anydoc. Copy 0.1.8 and re-export +# that one function so anydoc-go can keep the official serializer. +prepare_patched_anydoc() { + local dest="$crate_dir/patched-anydoc" + if [ -f "$dest/.weknora-patched" ]; then + return + fi + + local src="" + local cargo_home="${CARGO_HOME:-$HOME/.cargo}" + src=$(find "$cargo_home/registry/src" -maxdepth 2 -type d -name 'anydoc-0.1.8' 2>/dev/null | head -1 || true) + + if [ -z "$src" ]; then + local tarball="$crate_dir/.anydoc-0.1.8.crate" + echo "Fetching anydoc 0.1.8 to patch document_to_markdown into the public API" + if ! curl -fsSL "https://rsproxy.cn/api/v1/crates/anydoc/0.1.8/download" -o "$tarball"; then + curl -fsSL "https://static.crates.io/crates/anydoc/anydoc-0.1.8.crate" -o "$tarball" + fi + local unpack="$crate_dir/.anydoc-unpack" + rm -rf "$unpack" + mkdir -p "$unpack" + tar -xzf "$tarball" -C "$unpack" + src=$(find "$unpack" -maxdepth 1 -type d -name 'anydoc-0.1.8' | head -1) + if [ -z "$src" ]; then + echo "error: unpacked anydoc-0.1.8 crate is missing" >&2 + exit 1 + fi + fi + + rm -rf "$dest" + cp -R "$src" "$dest" + if ! grep -q '^use render::markdown::document_to_markdown;$' "$dest/src/lib.rs"; then + echo "error: anydoc 0.1.8 lib.rs no longer has the expected document_to_markdown import" >&2 + exit 1 + fi + sed -i.bak 's/^use render::markdown::document_to_markdown;/pub use render::markdown::document_to_markdown;/' "$dest/src/lib.rs" + rm -f "$dest/src/lib.rs.bak" + touch "$dest/.weknora-patched" + rm -rf "$crate_dir/.anydoc-unpack" "$crate_dir/.anydoc-0.1.8.crate" +} + +# --locked: build exactly the dependency versions in the committed Cargo.lock. +# That lockfile is what pins the patched pdf-inspector/lopdf, so a silent +# resolver drift must fail the build rather than ship an unaudited tree. +echo "Building anydoc archive for $target" +prepare_patched_anydoc +cargo build --release --locked --manifest-path "$crate_dir/Cargo.toml" --target "$target" + +dest="$crate_dir/lib/$lib_dir" +mkdir -p "$dest" +cp "$crate_dir/target/$target/release/$lib_name" "$dest/$lib_name" + +echo "Wrote $dest/$lib_name" +echo "Build WeKnora with the archive: go build -tags anydoc ./cmd/server" diff --git a/scripts/dev.sh b/scripts/dev.sh index f76c6688..d3836a55 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -406,6 +406,46 @@ check_remote_dev_connectivity() { return 0 } +# Host-platform path of the anydoc static archive (built by `make anydoc-lib`). +anydoc_host_archive() { + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_arm64/libanydoc_go.a" ;; + Darwin-x86_64) echo "$PROJECT_ROOT/third_party/anydoc-go/lib/darwin_amd64/libanydoc_go.a" ;; + Linux-x86_64) + if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a" ]; then + echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_gnu/libanydoc_go.a" + else + echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_amd64_musl/libanydoc_go.a" + fi + ;; + Linux-aarch64) + if [ -f "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a" ]; then + echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_gnu/libanydoc_go.a" + else + echo "$PROJECT_ROOT/third_party/anydoc-go/lib/linux_arm64_musl/libanydoc_go.a" + fi + ;; + *) echo "" ;; + esac +} + +# Enable the in-process anydoc engine when the archive is present, unless the +# caller already set GO_BUILD_TAGS (including empty, which opts out). +enable_anydoc_build_tag() { + if [ -n "${GO_BUILD_TAGS+x}" ]; then + export GO_BUILD_TAGS + return + fi + local archive + archive="$(anydoc_host_archive)" + if [ -n "$archive" ] && [ -f "$archive" ]; then + export GO_BUILD_TAGS=anydoc + log_info "检测到 anydoc 静态库,已启用 -tags anydoc" + else + log_info "未检测到 anydoc 静态库,解析引擎不可用。需要时先运行: make anydoc-lib" + fi +} + # 启动后端应用(本地) start_app() { log_info "启动后端应用(本地开发模式)..." @@ -477,6 +517,8 @@ start_app() { export CGO_LDFLAGS="-Wl,-no_warn_duplicate_libraries" fi + enable_anydoc_build_tag + # 检查是否安装了 Air(热重载工具) if command -v air &> /dev/null; then log_success "检测到 Air,使用热重载模式启动..." @@ -487,7 +529,7 @@ start_app() { log_warning "提示: 安装 Air 可以实现代码修改后自动重启" log_info "安装命令: go install github.com/air-verse/air@latest" LDFLAGS="$(./scripts/get_version.sh ldflags) -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'" - go run -ldflags="$LDFLAGS" ./cmd/server + go run -tags "${GO_BUILD_TAGS:-}" -ldflags="$LDFLAGS" ./cmd/server fi } diff --git a/third_party/anydoc-go/.gitignore b/third_party/anydoc-go/.gitignore new file mode 100644 index 00000000..a5a68772 --- /dev/null +++ b/third_party/anydoc-go/.gitignore @@ -0,0 +1,3 @@ +target/ +lib/ +patched-anydoc/ diff --git a/third_party/anydoc-go/Cargo.lock b/third_party/anydoc-go/Cargo.lock new file mode 100644 index 00000000..4425fa1a --- /dev/null +++ b/third_party/anydoc-go/Cargo.lock @@ -0,0 +1,1526 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anydoc" +version = "0.1.8" +dependencies = [ + "calamine", + "cfb", + "csv", + "encoding_rs", + "flate2", + "log", + "pdf-inspector", + "quick-xml", + "zip", +] + +[[package]] +name = "anydoc-go" +version = "0.1.8" +dependencies = [ + "anydoc", + "cbindgen", +] + +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cbindgen" +version = "0.29.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ecb53484c9c167ba674026b656d8a27d7657a58e6066aa902bfb1a4aa00ae20" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", + "tempfile", + "toml", +] + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom", + "indexmap", + "itoa", + "jiff", + "log", + "md-5", + "nom", + "rand", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror", + "time", + "ttf-parser", + "weezl", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pdf-inspector" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf", + "once_cell", + "rayon", + "regex", + "thiserror", + "ttf-parser", + "unicode-normalization", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/third_party/anydoc-go/Cargo.toml b/third_party/anydoc-go/Cargo.toml new file mode 100644 index 00000000..433f6961 --- /dev/null +++ b/third_party/anydoc-go/Cargo.toml @@ -0,0 +1,39 @@ +# Go bindings for anydoc. +# +# `crate-type = ["staticlib"]` produces libanydoc_go.a, which the Go module +# links at build time via cgo. Not a cdylib: cgo wants an archive it can pull +# symbols from, not a loadable shared object. +[package] +name = "anydoc-go" +version = "0.1.8" +edition = "2024" +description = "Go bindings (C ABI) for anydoc" +license = "MIT" +repository = "https://github.com/firecrawl/anydoc" +publish = false + +[lib] +crate-type = ["staticlib"] + +# Standalone crate: it lives inside the WeKnora tree, not the anydoc +# workspace, so it takes the published crate rather than a path dependency. +# Pinned exactly: the C ABI mirrors this version's error and format enums. +[dependencies] +anydoc = "=0.1.8" + +[build-dependencies] +cbindgen = "0.29" + +# document_to_markdown is crate-private in published anydoc. The build script +# copies 0.1.8 and re-exports that one function so we can keep the official +# GFM serializer after rewriting asset images to External URLs. +[patch.crates-io] +anydoc = { path = "patched-anydoc" } + +[workspace] + +# Mirrors the anydoc workspace's release profile: without it the archive keeps +# every debug symbol and roughly doubles in size. +[profile.release] +lto = "thin" +strip = "symbols" diff --git a/third_party/anydoc-go/LICENSE b/third_party/anydoc-go/LICENSE new file mode 100644 index 00000000..7ad9f5ab --- /dev/null +++ b/third_party/anydoc-go/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sideguide Technologies Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/anydoc-go/README.md b/third_party/anydoc-go/README.md new file mode 100644 index 00000000..8eff10eb --- /dev/null +++ b/third_party/anydoc-go/README.md @@ -0,0 +1,128 @@ +# anydoc-go (vendored) + +Go bindings for [anydoc](https://github.com/firecrawl/anydoc), the Rust library +that converts Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF +documents to GitHub-Flavored Markdown. WeKnora links them through cgo so office +documents can be parsed inside the Go process, without the Python docreader +service. + +## Why this is vendored + +The bindings come from the open pull request +[firecrawl/anydoc#30](https://github.com/firecrawl/anydoc/pull/30). Until it +merges and upstream tags `go/vX.Y.Z`, there is no module to `go get`: the PR +also expects a maintainer-only workflow to commit per-platform archives that do +not exist yet. So the binding source lives here and `go.mod` points at it: + +``` +replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go +``` + +When the upstream module is published, delete this directory, drop the +`replace`, and require the published version. Nothing else in WeKnora changes: +only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it. + +## Provenance + +| | | +| --- | --- | +| Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") | +| PR head | `1a7a6c0` | +| Rebased onto | `4e3089b` (`chore: release v0.1.8`) | +| anydoc crate | `0.1.8` (from crates.io, pinned with `=`) | +| License | MIT (see LICENSE) | + +The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before +vendoring. The rebase conflicts were all release bookkeeping — the workspace +member list, the README binding sections, and the version-agreement script, +which the wasm bindings had touched in the meantime — plus the binding's own +version, bumped from 0.1.3 to 0.1.8. Taking v0.1.8 also picks up the +`pdf-inspector` 0.1.8 bump that fixes [RUSTSEC-2026-0187](https://rustsec.org/advisories/RUSTSEC-2026-0187.html), +the `lopdf` stack overflow that aborts the process on a hostile PDF. + +## Local modifications + +Keep this list current: it is the diff a future upgrade has to re-apply. Items +2–4 are bugs in the upstream PR and are worth sending back to it. + +1. `Cargo.toml` — depends on the published `anydoc = "=0.1.8"` crate instead of + the workspace path dependency, declares its own empty `[workspace]`, and + repeats the upstream release profile (`lto`, `strip`), which it would + otherwise inherit from the anydoc workspace. +2. `anydoc.go` — every ABI call runs inside `call()`, which pins the goroutine + with `runtime.LockOSThread` for the duration. The ABI reports the error + message through a thread-local slot that a *second* call + (`anydoc_last_error`) reads, and Go may resume a goroutine on a different OS + thread once a cgo call returns: without pinning, a failed conversion reports + an empty message, or one belonging to another document parsed on the thread + it landed on. Reproduced at roughly 1 in 1500 concurrent conversions; the + regression test is `TestErrorDetailSurvivesConcurrency` in + `internal/infrastructure/docparser/anydoc`. +3. `model.go` — decoder preallocations are bounded by the bytes left in the + buffer (`capFor`), and `need` rejects a negative length. A count taken + straight from the buffer is only trustworthy while the Rust encoder and this + decoder agree; on a skew, `make([]Block, 0, n)` would exhaust memory before + the first bounds check, turning a version mismatch into a dead process. +4. `src/lib.rs` — the three conversion entry points run inside `guarded()`, + which catches a panic and reports it as a malformed document. A panic + escaping an `extern "C"` function aborts the process, and WeKnora parses + untrusted uploads in the same process that serves the API. Note the limit: + this cannot contain a stack overflow or an allocation failure, which is why + the dependency pin below matters as much as the guard. +5. Removed the upstream CLI (`cmd/anydoc`) and the binding test suite, which + reads fixtures from the anydoc repository. WeKnora's own tests live in + `internal/infrastructure/docparser/anydoc`. +6. `scripts/build-anydoc-lib.sh` copies anydoc 0.1.8 to `patched-anydoc/` + (gitignored) and re-exports `document_to_markdown`, which is crate-private + in the published crate. `src/asset_links.rs` then rewrites `ImageSource::Asset` + to `External("images/image-N.ext")` so the official serializer emits in-place + image links. `anydoc_to_markdown_with_asset_links` is the ABI for that path. + +## Dependency pinning and audit + +`Cargo.lock` is committed and `scripts/build-anydoc-lib.sh` builds with +`--locked`, so the archive is always the audited dependency tree. CI runs +`cargo audit` against it, because the crate that fails here is the one parsing +untrusted uploads inside the API process. + +That matters concretely: with `lopdf` 0.41 — what the upstream PR's own +lockfile resolved to — a ~100 KB PDF holding a deeply nested catalog array +kills the process with a stack overflow (`RUSTSEC-2026-0187`), which neither +`guarded()` nor Go's `recover` can contain. anydoc 0.1.8 moved to `lopdf` 0.42 +and the same input comes back as an ordinary error; +`TestDeeplyNestedPDFFailsWithoutKillingTheProcess` keeps it that way. + +`cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is +unmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. It +is not a vulnerability and nothing here can fix it, so warnings report without +failing the job. + +## Known upstream limitation + +Markdown rendering drops embedded images: `ImageSource::Asset` renders as its +alt text, and the bytes are only reachable through the document model. The +renderer (`document_to_markdown`) is also crate-private. WeKnora keeps that +serializer: `scripts/build-anydoc-lib.sh` re-exports the one function, then +`anydoc_to_markdown_with_asset_links` rewrites `Asset` images to +`ImageSource::External("images/image-N.ext")` so the official GFM output +places them in reading order. PDF has no document model; scanned pages fall +back to the builtin docreader so they can be rasterized for OCR. + +## Building the archive + +cgo links `lib//libanydoc_go.a`, which is a build artifact (~30 MB) +and is therefore git-ignored rather than committed: + +```bash +scripts/build-anydoc-lib.sh # host platform +TARGET=aarch64-unknown-linux-musl scripts/build-anydoc-lib.sh +``` + +Then build WeKnora with the engine linked in: + +```bash +make build-anydoc # or: go build -tags anydoc ./cmd/server +``` + +Builds without the `anydoc` tag need no Rust toolchain and no archive; the +engine simply reports itself as unavailable. diff --git a/third_party/anydoc-go/anydoc.go b/third_party/anydoc-go/anydoc.go new file mode 100644 index 00000000..5830e758 --- /dev/null +++ b/third_party/anydoc-go/anydoc.go @@ -0,0 +1,195 @@ +// Package anydoc converts documents (Word, PowerPoint, Excel, OpenDocument, +// RTF, EPUB, CSV, and PDF) to GitHub-Flavored Markdown, with full access to +// the parsed document model and embedded assets. +// +// This is the Go binding. It links a Rust static library through cgo; the +// module packages archives for every supported platform. See the README for +// the supported platform matrix and musl build tag. +package anydoc + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/lib/darwin_arm64 -lanydoc_go -lm -lstdc++ +#cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/lib/darwin_amd64 -lanydoc_go -lm -lstdc++ +#cgo windows,amd64 LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -lanydoc_go + +#include +#include +#include +#include "include/anydoc.h" +*/ +import "C" + +import ( + "fmt" + "runtime" + "unsafe" +) + +// call runs one ABI entry point and turns its status code into an error. +// +// The goroutine is pinned for the duration because the ABI reports the error +// message through a thread-local slot that a *second* call +// (`anydoc_last_error`) reads. Go is free to resume a goroutine on a different +// OS thread after a cgo call returns, and a conversion is long enough for that +// to happen: the message would then be missing, or belong to another +// conversion that ran on the thread we landed on. +func call(entry func() C.int) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if code := entry(); code != errOK { + return convertError(code) + } + return nil +} + +// ToMarkdown converts a document file to Markdown. The format is detected +// from the file content; the extension is the fallback for signature-less +// formats (CSV) and unrecognizable containers. +func ToMarkdown(path string) (string, error) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + var out *C.char + var outLen C.uintptr_t + if err := call(func() C.int { return C.anydoc_to_markdown(cpath, &out, &outLen) }); err != nil { + return "", err + } + defer C.anydoc_string_free(out) + markdown, err := cStringN(out, outLen) + if err != nil { + return "", err + } + return markdown, nil +} + +// ToMarkdownBytes converts an in-memory document to Markdown. Pass a Format +// to select the parser, or nil to detect it from the content, which +// signature-less formats (CSV) have to name explicitly. +func ToMarkdownBytes(data []byte, format *Format) (string, error) { + if len(data) == 0 { + return "", &ConvertError{Kind: "unsupported", Detail: "empty input"} + } + tag := C.int(C.ANYDOC_FORMAT_NONE) + if format != nil { + tag = formatToTag(*format) + if tag == C.int(C.ANYDOC_FORMAT_NONE) { + return "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)} + } + } + var out *C.char + var outLen C.uintptr_t + if err := call(func() C.int { + return C.anydoc_to_markdown_bytes( + (*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &out, &outLen, + ) + }); err != nil { + return "", err + } + defer C.anydoc_string_free(out) + markdown, err := cStringN(out, outLen) + if err != nil { + return "", err + } + return markdown, nil +} + +// ToMarkdownWithAssetLinks converts an in-memory document to Markdown with +// embedded images rewritten as `![alt](images/image-N.ext)` so they keep +// their original positions. The Markdown itself is produced by anydoc's +// official GFM serializer. +func ToMarkdownWithAssetLinks(data []byte, format *Format) (string, error) { + if len(data) == 0 { + return "", &ConvertError{Kind: "unsupported", Detail: "empty input"} + } + tag := C.int(C.ANYDOC_FORMAT_NONE) + if format != nil { + tag = formatToTag(*format) + if tag == C.int(C.ANYDOC_FORMAT_NONE) { + return "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)} + } + } + var out *C.char + var outLen C.uintptr_t + if err := call(func() C.int { + return C.anydoc_to_markdown_with_asset_links( + (*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &out, &outLen, + ) + }); err != nil { + return "", err + } + defer C.anydoc_string_free(out) + markdown, err := cStringN(out, outLen) + if err != nil { + return "", err + } + return markdown, nil +} + +// ToDocument parses an in-memory document into the document model, which also +// carries the embedded assets. Pass a Format to select the parser, or nil to +// detect it from the content. +// +// Unsupported for PDF: PDF conversion produces Markdown directly and has no +// document-model form; use ToMarkdownBytes. +func ToDocument(data []byte, format *Format) (*Document, error) { + if len(data) == 0 { + return nil, &ConvertError{Kind: "unsupported", Detail: "empty input"} + } + tag := C.int(C.ANYDOC_FORMAT_NONE) + if format != nil { + tag = formatToTag(*format) + if tag == C.int(C.ANYDOC_FORMAT_NONE) { + return nil, &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)} + } + } + var buf *C.uint8_t + var bufLen C.uintptr_t + if err := call(func() C.int { + return C.anydoc_to_document( + (*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &buf, &bufLen, + ) + }); err != nil { + return nil, err + } + defer C.anydoc_buffer_free(buf, bufLen) + if buf == nil || bufLen == 0 { + return &Document{}, nil + } + // Copy the C buffer into a Go-owned slice before decoding, so the C + // buffer can be freed immediately rather than tying its lifetime to the + // decoded Document. + raw, err := cGoBytes(unsafe.Pointer(buf), bufLen) + if err != nil { + return nil, err + } + return decodeDocument(raw) +} + +// cStringN reads a length-prefixed C buffer as a Go string. The bytes are +// copied, so the caller may free the C buffer immediately after. +func cStringN(s *C.char, n C.uintptr_t) (string, error) { + if s == nil || n == 0 { + return "", nil + } + b, err := cGoBytes(unsafe.Pointer(s), n) + if err != nil { + return "", err + } + return string(b), nil +} + +func cGoBytes(ptr unsafe.Pointer, n C.uintptr_t) ([]byte, error) { + // C.GoBytes accepts a C int length even on 64-bit Go builds. + if uint64(n) > 1<<31-1 { + return nil, &ConvertError{Kind: "resource_limit", Detail: fmt.Sprintf("FFI output is too large for Go: %d bytes", n)} + } + return C.GoBytes(ptr, C.int(n)), nil +} + +// cString reads a NUL-terminated C string as a Go string. +func cString(s *C.char) string { + if s == nil { + return "" + } + return C.GoString(s) +} diff --git a/third_party/anydoc-go/build.rs b/third_party/anydoc-go/build.rs new file mode 100644 index 00000000..09bb51ba --- /dev/null +++ b/third_party/anydoc-go/build.rs @@ -0,0 +1,38 @@ +//! Regenerate `include/anydoc.h` from the crate's C ABI on every build. +//! `cbindgen` is a build-dependency, so this runs whenever `cargo build +//! -p anydoc-go` runs inside the workspace. The committed header is the +//! source of truth for the Go side; consumers link the packaged static archive +//! and never run this. + +use std::path::PathBuf; + +fn main() { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let include_dir = manifest_dir.join("include"); + let header = include_dir.join("anydoc.h"); + + std::fs::create_dir_all(&include_dir).expect("create include directory"); + + let config = cbindgen::Config { + language: cbindgen::Language::C, + include_guard: Some("ANYDOC_H".to_string()), + autogen_warning: Some( + "/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */" + .to_string(), + ), + ..Default::default() + }; + + match cbindgen::Builder::new().with_crate(&manifest_dir).with_config(config).generate() { + Ok(bindings) => { + let _ = bindings.write_to_file(&header); + } + Err(err) => { + println!("cargo:warning=cbindgen failed to generate anydoc.h: {err}"); + } + } + + println!("cargo:rerun-if-changed=src/lib.rs"); + println!("cargo:rerun-if-changed=src/model.rs"); + println!("cargo:rerun-if-changed=src/asset_links.rs"); +} diff --git a/third_party/anydoc-go/errors.go b/third_party/anydoc-go/errors.go new file mode 100644 index 00000000..9000e837 --- /dev/null +++ b/third_party/anydoc-go/errors.go @@ -0,0 +1,95 @@ +package anydoc + +/* +#include +#include "include/anydoc.h" +*/ +import "C" + +import "fmt" + +// Error codes mirroring the C ABI (ERR_*). Stable across versions. +const ( + errOK = C.ERR_OK + errUnsupported = C.ERR_UNSUPPORTED + errMalformed = C.ERR_MALFORMED + errEncrypted = C.ERR_ENCRYPTED + errResourceLimit = C.ERR_RESOURCE_LIMIT + errMissingPart = C.ERR_MISSING_PART + errIO = C.ERR_IO + errPDFNoModel = C.ERR_PDF_NO_MODEL + errInvalidArg = C.ERR_INVALID_ARG + errUnknownFormat = C.ERR_UNKNOWN_FORMAT +) + +// ConvertError is the typed error every conversion function returns. It +// carries the same variant kind the Node and Python bindings expose +// ("unsupported", "malformed", "encrypted", ...) plus the crate's +// human-readable detail. +type ConvertError struct { + // Kind is the lowercase variant name, matching the Node and Python + // bindings: "unsupported", "malformed", "encrypted", "resource_limit", + // "missing_part", "io", "pdf_no_model". Go also reports + // "unknown_format" for an invalid explicit Format. + Kind string + // Detail is the crate's Display output for the error. + Detail string +} + +func (e *ConvertError) Error() string { + if e == nil { + return "" + } + if e.Detail == "" { + return "anydoc: " + e.Kind + } + return "anydoc: " + e.Kind + ": " + e.Detail +} + +// convertError builds a ConvertError from an ABI error code, pulling the +// detail message from the thread-local last-error slot. +func convertError(code C.int) error { + if code == errOK { + return nil + } + kind := errorKind(code) + detail := lastError() + return &ConvertError{Kind: kind, Detail: detail} +} + +func errorKind(code C.int) string { + switch code { + case errUnsupported: + return "unsupported" + case errMalformed: + return "malformed" + case errEncrypted: + return "encrypted" + case errResourceLimit: + return "resource_limit" + case errMissingPart: + return "missing_part" + case errIO: + return "io" + case errPDFNoModel: + return "pdf_no_model" + case errInvalidArg: + return "invalid_argument" + case errUnknownFormat: + return "unknown_format" + default: + return fmt.Sprintf("unknown_error(%d)", int(code)) + } +} + +// lastError returns the human-readable message the last ABI call stashed on +// the current OS thread, or "" when there was none. The C side returns a +// freshly allocated string the caller must free. +func lastError() string { + ptr := C.anydoc_last_error() + if ptr == nil { + return "" + } + defer C.anydoc_string_free(ptr) + return cString(ptr) +} diff --git a/third_party/anydoc-go/format.go b/third_party/anydoc-go/format.go new file mode 100644 index 00000000..05bed7c8 --- /dev/null +++ b/third_party/anydoc-go/format.go @@ -0,0 +1,143 @@ +package anydoc + +/* +#include "include/anydoc.h" +*/ +import "C" + +import ( + "strings" + "unsafe" +) + +// Format names a document format. The string values are the lowercase +// extension names, matching the Node and Python bindings ("doc", "docx", +// "pdf", ...). Use the exported constants rather than raw strings where +// possible: the constants are pinned to the C ABI's tag values at build +// time. +type Format string + +// Format constants. Every value is stable across versions. +const ( + FormatDoc Format = "doc" + FormatDocx Format = "docx" + FormatOdt Format = "odt" + FormatPdf Format = "pdf" + FormatPpt Format = "ppt" + FormatPptx Format = "pptx" + FormatRtf Format = "rtf" + FormatEpub Format = "epub" + FormatXlsx Format = "xlsx" + FormatOds Format = "ods" + FormatOdp Format = "odp" + FormatCsv Format = "csv" +) + +// formatToTag maps a Format to its C ABI tag. Returns ANYDOC_FORMAT_NONE for +// an unknown format. +func formatToTag(f Format) C.int { + switch f { + case FormatDoc: + return C.int(C.ANYDOC_FORMAT_DOC) + case FormatDocx: + return C.int(C.ANYDOC_FORMAT_DOCX) + case FormatOdt: + return C.int(C.ANYDOC_FORMAT_ODT) + case FormatPdf: + return C.int(C.ANYDOC_FORMAT_PDF) + case FormatPpt: + return C.int(C.ANYDOC_FORMAT_PPT) + case FormatPptx: + return C.int(C.ANYDOC_FORMAT_PPTX) + case FormatRtf: + return C.int(C.ANYDOC_FORMAT_RTF) + case FormatEpub: + return C.int(C.ANYDOC_FORMAT_EPUB) + case FormatXlsx: + return C.int(C.ANYDOC_FORMAT_XLSX) + case FormatOds: + return C.int(C.ANYDOC_FORMAT_ODS) + case FormatOdp: + return C.int(C.ANYDOC_FORMAT_ODP) + case FormatCsv: + return C.int(C.ANYDOC_FORMAT_CSV) + } + return C.int(C.ANYDOC_FORMAT_NONE) +} + +// tagToFormat maps a C ABI tag back to a Format. ok is false for NONE or an +// unknown tag. +func tagToFormat(tag C.int) (Format, bool) { + switch tag { + case C.ANYDOC_FORMAT_DOC: + return FormatDoc, true + case C.ANYDOC_FORMAT_DOCX: + return FormatDocx, true + case C.ANYDOC_FORMAT_ODT: + return FormatOdt, true + case C.ANYDOC_FORMAT_PDF: + return FormatPdf, true + case C.ANYDOC_FORMAT_PPT: + return FormatPpt, true + case C.ANYDOC_FORMAT_PPTX: + return FormatPptx, true + case C.ANYDOC_FORMAT_RTF: + return FormatRtf, true + case C.ANYDOC_FORMAT_EPUB: + return FormatEpub, true + case C.ANYDOC_FORMAT_XLSX: + return FormatXlsx, true + case C.ANYDOC_FORMAT_ODS: + return FormatOds, true + case C.ANYDOC_FORMAT_ODP: + return FormatOdp, true + case C.ANYDOC_FORMAT_CSV: + return FormatCsv, true + } + return "", false +} + +// FormatFromBytes detects the format from the content itself: the signature +// and identity each container specification designates (PDF header, RTF open +// group, OLE stream names, ZIP package mimetype/content types). Plain-text +// formats (CSV) carry no signature and return the zero Format and ok=false; +// so does anything unrecognized. +func FormatFromBytes(data []byte) (f Format, ok bool) { + if len(data) == 0 { + return "", false + } + var tag C.int + code := C.anydoc_format_from_bytes((*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} + +// FormatFromExtension returns the format a bare extension names (with or +// without a leading dot), matched case-insensitively. ok is false for +// anything unrecognized. +func FormatFromExtension(ext string) (f Format, ok bool) { + ext = strings.TrimPrefix(ext, ".") + cext := C.CString(strings.ToLower(ext)) + defer C.free(unsafe.Pointer(cext)) + var tag C.int + code := C.anydoc_format_from_extension(cext, &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} + +// FormatFromPath returns the format a path's extension names. ok is false +// when the path has no extension or names nothing recognized. +func FormatFromPath(path string) (f Format, ok bool) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + var tag C.int + code := C.anydoc_format_from_path(cpath, &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} diff --git a/third_party/anydoc-go/go.mod b/third_party/anydoc-go/go.mod new file mode 100644 index 00000000..d0780b92 --- /dev/null +++ b/third_party/anydoc-go/go.mod @@ -0,0 +1,3 @@ +module github.com/firecrawl/anydoc/go + +go 1.22 diff --git a/third_party/anydoc-go/include/anydoc.h b/third_party/anydoc-go/include/anydoc.h new file mode 100644 index 00000000..179ff5ce --- /dev/null +++ b/third_party/anydoc-go/include/anydoc.h @@ -0,0 +1,264 @@ +#ifndef ANYDOC_H +#define ANYDOC_H + +/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */ + +#include +#include +#include +#include + +/** + * Success. + */ +#define ERR_OK 0 + +/** + * `ConvertError::Unsupported`. + */ +#define ERR_UNSUPPORTED 1 + +/** + * `ConvertError::Malformed`. + */ +#define ERR_MALFORMED 2 + +/** + * `ConvertError::Encrypted`. + */ +#define ERR_ENCRYPTED 3 + +/** + * `ConvertError::ResourceLimit`. + */ +#define ERR_RESOURCE_LIMIT 4 + +/** + * `ConvertError::MissingPart`. + */ +#define ERR_MISSING_PART 5 + +/** + * `ConvertError::Io`. + */ +#define ERR_IO 6 + +/** + * PDF was passed to `anydoc_to_document` (unsupported there). + */ +#define ERR_PDF_NO_MODEL 7 + +/** + * Null pointer or invalid argument passed to the ABI. + */ +#define ERR_INVALID_ARG 8 + +/** + * Unknown format name passed to the ABI. + */ +#define ERR_UNKNOWN_FORMAT 9 + +/** + * C-side format tag. Stable; mirrors the Node/Python lowercase string names + * via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel. + */ +#define ANYDOC_FORMAT_NONE -1 + +#define ANYDOC_FORMAT_DOC 0 + +#define ANYDOC_FORMAT_DOCX 1 + +#define ANYDOC_FORMAT_ODT 2 + +#define ANYDOC_FORMAT_PDF 3 + +#define ANYDOC_FORMAT_PPT 4 + +#define ANYDOC_FORMAT_PPTX 5 + +#define ANYDOC_FORMAT_RTF 6 + +#define ANYDOC_FORMAT_EPUB 7 + +#define ANYDOC_FORMAT_XLSX 8 + +#define ANYDOC_FORMAT_ODS 9 + +#define ANYDOC_FORMAT_ODP 10 + +#define ANYDOC_FORMAT_CSV 11 + +/** + * Tag values for `Block` variants. Stable across versions; the Go side maps + * these to the same lowercase string names the Node and Python bindings use. + */ +#define BLOCK_HEADING 0 + +#define BLOCK_PARAGRAPH 1 + +#define BLOCK_LIST 2 + +#define BLOCK_TABLE 3 + +#define BLOCK_QUOTE 4 + +#define BLOCK_CODE 5 + +#define BLOCK_RULE 6 + +#define INLINE_TEXT 0 + +#define INLINE_LINK 1 + +#define INLINE_IMAGE 2 + +#define INLINE_ANCHOR 3 + +#define INLINE_NOTEREF 4 + +#define INLINE_LINEBREAK 5 + +#define LINK_EXTERNAL 0 + +#define LINK_RELATIVE 1 + +#define LINK_ANCHOR 2 + +#define IMG_EXTERNAL 0 + +#define IMG_ASSET 1 + +#define IMG_UNAVAILABLE 2 + +#define MARKER_BULLET 0 + +#define MARKER_DECIMAL 1 + +#define MARKER_LOWER_ALPHA 2 + +#define MARKER_UPPER_ALPHA 3 + +#define MARKER_LOWER_ROMAN 4 + +#define MARKER_UPPER_ROMAN 5 + +#define TABLE_DATA 0 + +#define TABLE_LAYOUT 1 + +#define SLOT_ORIGIN 0 + +#define SLOT_COVERED 1 + +#define NOTE_FOOTNOTE 0 + +#define NOTE_ENDNOTE 1 + +/** + * Detect the format from the content. Writes a format tag to `*out` (one of + * `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches. + * + * Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null + * pointers. + */ +int anydoc_format_from_bytes(const uint8_t *bytes, uintptr_t len, int *out); + +/** + * The format a bare extension names (no leading dot). Writes a format tag to + * `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized. + */ +int anydoc_format_from_extension(const char *ext, int *out); + +/** + * The format a path's extension names. Writes a format tag to `*out`, or + * `ANYDOC_FORMAT_NONE` when unrecognized. + */ +int anydoc_format_from_path(const char *path, int *out); + +/** + * Convert a document file to Markdown. The format is detected from the file + * content, with the extension as the fallback for signature-less formats + * (CSV). + * + * On success writes a `*mut c_char` to `*out_str` and its byte length to + * `*out_len`. The caller must free the string with `anydoc_string_free`. + */ +int anydoc_to_markdown(const char *path, char **out_str, uintptr_t *out_len); + +/** + * Convert an in-memory document to Markdown. `format_tag` is one of the + * `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the + * content. + */ +int anydoc_to_markdown_bytes(const uint8_t *bytes, + uintptr_t len, + int format_tag, + char **out_str, + uintptr_t *out_len); + +/** + * Convert an in-memory document to Markdown with embedded images rewritten + * as `![alt](images/image-N.ext)` so they keep their original positions. + * + * This is `to_document` + anydoc's official GFM serializer. Asset images are + * first turned into `ImageSource::External` URLs; the serializer already + * emits those as ordinary Markdown image links. PDF has no document model + * and is converted the same way as `anydoc_to_markdown_bytes`. + */ +int anydoc_to_markdown_with_asset_links(const uint8_t *bytes, + uintptr_t len, + int format_tag, + char **out_str, + uintptr_t *out_len); + +/** + * Parse an in-memory document into the document model. `format_tag` is one of + * the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the + * content. + * + * On success writes a heap-allocated byte buffer to `*out_buf` and its length + * to `*out_len`. The buffer holds the flat serialization (see `model`). The + * caller must free it with `anydoc_buffer_free`. + * + * Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown + * directly and has no document-model form. Use `anydoc_to_markdown_bytes`. + */ +int anydoc_to_document(const uint8_t *bytes, + uintptr_t len, + int format_tag, + uint8_t **out_buf, + uintptr_t *out_len); + +/** + * Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`. + * Null is a no-op (the empty-string sentinel). + */ +void anydoc_string_free(char *s); + +/** + * Free a document buffer returned by `anydoc_to_document`. Null is a no-op. + * `len` must be the length written to `*out_len` at allocation time; it is + * validated against the header for safety. + */ +void anydoc_buffer_free(uint8_t *buf, uintptr_t len); + +/** + * Return the human-readable message for the last error on the current + * thread. The returned `*mut c_char` is freshly allocated and must be freed + * with `anydoc_string_free`. Returns null when there is no last error. + */ +char *anydoc_last_error(void); + +/** + * Return the lowercase name of a format tag, or null for an unknown tag. + * The returned string must be freed with `anydoc_string_free`. + */ +char *anydoc_format_name(int tag); + +/** + * Parse a lowercase format name into a tag. Writes the tag to `*out`, or + * `ANYDOC_FORMAT_NONE` when the name is unknown. + */ +int anydoc_format_from_name(const char *name, int *out); + +#endif /* ANYDOC_H */ diff --git a/third_party/anydoc-go/lib_linux_gnu.go b/third_party/anydoc-go/lib_linux_gnu.go new file mode 100644 index 00000000..1618280e --- /dev/null +++ b/third_party/anydoc-go/lib_linux_gnu.go @@ -0,0 +1,9 @@ +//go:build linux && !musl + +package anydoc + +/* +#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread +#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread +*/ +import "C" diff --git a/third_party/anydoc-go/lib_linux_musl.go b/third_party/anydoc-go/lib_linux_musl.go new file mode 100644 index 00000000..5c710676 --- /dev/null +++ b/third_party/anydoc-go/lib_linux_musl.go @@ -0,0 +1,9 @@ +//go:build linux && musl + +package anydoc + +/* +#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread +#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread +*/ +import "C" diff --git a/third_party/anydoc-go/model.go b/third_party/anydoc-go/model.go new file mode 100644 index 00000000..c0656950 --- /dev/null +++ b/third_party/anydoc-go/model.go @@ -0,0 +1,729 @@ +package anydoc + +/* +#include "include/anydoc.h" +*/ +import "C" + +import "fmt" + +// Document is a parsed document: its body, its notes, and the bytes of +// everything it embedded. A Document is self-contained — embedded assets +// carry their bytes, so it stays usable after the source archive is gone. +type Document struct { + Blocks []Block + // Notes are footnote and endnote bodies, referenced from text by an + // Inline with Kind == "note_ref". + Notes []Note + // Assets holds every embedded asset, indexed by Asset.ID as referenced + // by an Inline image source. + Assets []Asset +} + +// Block is one block-level piece of a document body. The Kind field selects +// which of the optional fields is populated; the rest are nil. +type Block struct { + // "heading", "paragraph", "list", "table", "block_quote", "code_block", + // or "rule". + Kind string + Level *uint8 // heading: 1-based outline depth + Anchor *string // heading: stable anchor id when the document targets it + Content []Inline // heading, paragraph + List *List // list + Table *Table // table + Blocks []Block // block_quote + Lang *string // code_block + Text *string // code_block +} + +// Inline is one span of inline content. The Kind field selects which optional +// fields are populated. +type Inline struct { + // "text", "link", "image", "anchor", "note_ref", or "line_break". + Kind string + Text *string // text + Style *Style // text + Content []Inline // link + Target *LinkTarget // link + Alt *string // image + Source *ImageSource // image + Anchor *string // anchor: the anchor id + NoteID *string // note_ref: the id of the note in Document.Notes +} + +// Style is a fully resolved character style. +type Style struct { + Bold bool + Italic bool + Strike bool + Code bool +} + +// LinkTarget is where a link points. +type LinkTarget struct { + // "external" (absolute URL with a scheme), "relative" (scheme-less + // relative reference, preserved as written), or "anchor" (internal + // target: a heading anchor or an "anchor" inline). + Kind string + Value string // the URL, relative reference, or anchor id +} + +// ImageSource is where an image's bytes live. +type ImageSource struct { + // "external" (absolute URL), "asset" (embedded image, carried in + // Document.Assets), or "unavailable" (no usable source; only the alt + // text remains). + Kind string + URL *string // external + AssetID *uint64 // asset: index into Document.Assets +} + +// List is a fully resolved list: numbering identity and marker resolution +// happen in the parser frontends. +type List struct { + // "bullet", "decimal", "lower_alpha", "upper_alpha", "lower_roman", or + // "upper_roman". + Marker string + Start uint64 + Items []ListItem +} + +// ListItem is one item of a List, which may hold nested blocks including +// further lists. +type ListItem struct { + Blocks []Block + Checked *bool // task-list state, when the item carries a checkbox + MarkerLabel *string // literal marker text overriding the level marker +} + +// Table is a canonical table grid: every logical grid position appears +// exactly once. Content and spans live on the origin slot, and each position +// a span covers holds a CellSlot with Kind == "covered" pointing back at that +// origin. +type Table struct { + Grid [][]CellSlot + HeaderRows uint32 // number of leading rows that are header rows (0 = none) + Kind string // "data" or "layout" +} + +// CellSlot is one position in a Table grid: either a cell or the shadow of +// one. +type CellSlot struct { + // "origin" (the cell itself) or "covered" (a position swallowed by a + // span, pointing back at the origin that covers it). + Kind string + Cell *Cell // origin + OriginRow *uint32 // covered: row of the covering origin + OriginCol *uint32 // covered: column of the covering origin +} + +// Cell is a table cell and the extent it spans. +type Cell struct { + Blocks []Block + ColSpan uint32 + RowSpan uint32 +} + +// Note is a footnote or endnote body, referenced from text by an Inline with +// Kind == "note_ref". +type Note struct { + ID string + Kind string // "footnote" or "endnote" + Blocks []Block +} + +// Asset is an embedded binary asset (image, object payload). Bytes are always +// retained, so a document stays self-contained. +type Asset struct { + ID uint64 // index into Document.Assets, as referenced by an image source + MediaType string + OriginPart string + Data []byte +} + +// ---- Decoding the flat FFI buffer ----------------------------------------- +// +// The buffer is the flat, length-prefixed serialization written by +// go/src/model.rs. Every field is little-endian. A decoder tracks the read +// cursor and reports a short or implausible buffer as an error — the Rust side +// always produces a well-formed buffer, so a short read is a bug or a version +// skew, not user input, and it must not take the process down. + +type decoder struct { + buf []byte + pos int +} + +func decodeDocument(raw []byte) (*Document, error) { + d := &decoder{buf: raw} + doc := &Document{} + var err error + if doc.Blocks, err = d.blocks(); err != nil { + return nil, err + } + if doc.Notes, err = d.notes(); err != nil { + return nil, err + } + if doc.Assets, err = d.assets(); err != nil { + return nil, err + } + return doc, nil +} + +// capFor bounds a preallocation by what is left in the buffer. Every element +// costs at least one byte to encode, so a count larger than the remaining +// bytes can only come from a corrupt buffer or a Rust/Go version skew — and +// preallocating it would exhaust memory before the first bounds check runs. +func (d *decoder) capFor(n uint32) int { + remaining := len(d.buf) - d.pos + if remaining < 0 { + return 0 + } + if uint64(n) > uint64(remaining) { + return remaining + } + return int(n) +} + +func (d *decoder) need(n int) error { + // A negative n means a length prefix overflowed int (32-bit builds); it is + // as unreadable as a truncated buffer. + if n < 0 || d.pos+n > len(d.buf) { + return fmt.Errorf("anydoc: truncated document buffer at offset %d (need %d bytes, have %d)", d.pos, n, len(d.buf)-d.pos) + } + return nil +} + +func (d *decoder) u8() (uint8, error) { + if err := d.need(1); err != nil { + return 0, err + } + v := d.buf[d.pos] + d.pos++ + return v, nil +} + +func (d *decoder) bool() (bool, error) { + v, err := d.u8() + if err != nil { + return false, err + } + return v != 0, nil +} + +func (d *decoder) u32() (uint32, error) { + if err := d.need(4); err != nil { + return 0, err + } + v := uint32(d.buf[d.pos]) | uint32(d.buf[d.pos+1])<<8 | uint32(d.buf[d.pos+2])<<16 | uint32(d.buf[d.pos+3])<<24 + d.pos += 4 + return v, nil +} + +func (d *decoder) u64() (uint64, error) { + if err := d.need(8); err != nil { + return 0, err + } + v := uint64(d.buf[d.pos]) | uint64(d.buf[d.pos+1])<<8 | uint64(d.buf[d.pos+2])<<16 | uint64(d.buf[d.pos+3])<<24 | + uint64(d.buf[d.pos+4])<<32 | uint64(d.buf[d.pos+5])<<40 | uint64(d.buf[d.pos+6])<<48 | uint64(d.buf[d.pos+7])<<56 + d.pos += 8 + return v, nil +} + +func (d *decoder) i32() (int32, error) { + u, err := d.u32() + if err != nil { + return 0, err + } + return int32(u), nil +} + +// optStr reads a length-prefixed string or a -1 sentinel (None). +func (d *decoder) optStr() (*string, error) { + length, err := d.i32() + if err != nil { + return nil, err + } + if length < 0 { + return nil, nil + } + s, err := d.strN(int(length)) + if err != nil { + return nil, err + } + return &s, nil +} + +func (d *decoder) str() (string, error) { + length, err := d.u32() + if err != nil { + return "", err + } + return d.strN(int(length)) +} + +func (d *decoder) strN(n int) (string, error) { + if err := d.need(n); err != nil { + return "", err + } + s := string(d.buf[d.pos : d.pos+n]) + d.pos += n + return s, nil +} + +func (d *decoder) bytes() ([]byte, error) { + length, err := d.u32() + if err != nil { + return nil, err + } + if err := d.need(int(length)); err != nil { + return nil, err + } + b := make([]byte, length) + copy(b, d.buf[d.pos:d.pos+int(length)]) + d.pos += int(length) + return b, nil +} + +func (d *decoder) blocks() ([]Block, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Block, 0, d.capFor(n)) + for i := uint32(0); i < n; i++ { + b, err := d.block() + if err != nil { + return nil, err + } + out = append(out, b) + } + return out, nil +} + +func (d *decoder) block() (Block, error) { + tag, err := d.i32() + if err != nil { + return Block{}, err + } + switch tag { + case C.BLOCK_HEADING: + level, err := d.u8() + if err != nil { + return Block{}, err + } + anchor, err := d.optStr() + if err != nil { + return Block{}, err + } + content, err := d.inlines() + if err != nil { + return Block{}, err + } + lv := level + return Block{Kind: "heading", Level: &lv, Anchor: anchor, Content: content}, nil + case C.BLOCK_PARAGRAPH: + content, err := d.inlines() + if err != nil { + return Block{}, err + } + return Block{Kind: "paragraph", Content: content}, nil + case C.BLOCK_LIST: + list, err := d.list() + if err != nil { + return Block{}, err + } + return Block{Kind: "list", List: list}, nil + case C.BLOCK_TABLE: + table, err := d.table() + if err != nil { + return Block{}, err + } + return Block{Kind: "table", Table: table}, nil + case C.BLOCK_QUOTE: + inner, err := d.blocks() + if err != nil { + return Block{}, err + } + return Block{Kind: "block_quote", Blocks: inner}, nil + case C.BLOCK_CODE: + lang, err := d.optStr() + if err != nil { + return Block{}, err + } + text, err := d.str() + if err != nil { + return Block{}, err + } + return Block{Kind: "code_block", Lang: lang, Text: &text}, nil + case C.BLOCK_RULE: + return Block{Kind: "rule"}, nil + default: + return Block{}, fmt.Errorf("anydoc: unknown block kind tag %d", tag) + } +} + +func (d *decoder) inlines() ([]Inline, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Inline, 0, d.capFor(n)) + for i := uint32(0); i < n; i++ { + in, err := d.inline() + if err != nil { + return nil, err + } + out = append(out, in) + } + return out, nil +} + +func (d *decoder) inline() (Inline, error) { + tag, err := d.i32() + if err != nil { + return Inline{}, err + } + switch tag { + case C.INLINE_TEXT: + text, err := d.str() + if err != nil { + return Inline{}, err + } + style, err := d.style() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "text", Text: &text, Style: &style}, nil + case C.INLINE_LINK: + content, err := d.inlines() + if err != nil { + return Inline{}, err + } + target, err := d.linkTarget() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "link", Content: content, Target: &target}, nil + case C.INLINE_IMAGE: + alt, err := d.str() + if err != nil { + return Inline{}, err + } + src, err := d.imageSource() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "image", Alt: &alt, Source: &src}, nil + case C.INLINE_ANCHOR: + id, err := d.str() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "anchor", Anchor: &id}, nil + case C.INLINE_NOTEREF: + id, err := d.str() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "note_ref", NoteID: &id}, nil + case C.INLINE_LINEBREAK: + return Inline{Kind: "line_break"}, nil + default: + return Inline{}, fmt.Errorf("anydoc: unknown inline kind tag %d", tag) + } +} + +func (d *decoder) style() (Style, error) { + bold, err := d.bool() + if err != nil { + return Style{}, err + } + italic, err := d.bool() + if err != nil { + return Style{}, err + } + strike, err := d.bool() + if err != nil { + return Style{}, err + } + code, err := d.bool() + if err != nil { + return Style{}, err + } + return Style{Bold: bold, Italic: italic, Strike: strike, Code: code}, nil +} + +func (d *decoder) linkTarget() (LinkTarget, error) { + tag, err := d.i32() + if err != nil { + return LinkTarget{}, err + } + value, err := d.str() + if err != nil { + return LinkTarget{}, err + } + switch tag { + case C.LINK_EXTERNAL: + return LinkTarget{Kind: "external", Value: value}, nil + case C.LINK_RELATIVE: + return LinkTarget{Kind: "relative", Value: value}, nil + case C.LINK_ANCHOR: + return LinkTarget{Kind: "anchor", Value: value}, nil + default: + return LinkTarget{}, fmt.Errorf("anydoc: unknown link target tag %d", tag) + } +} + +func (d *decoder) imageSource() (ImageSource, error) { + tag, err := d.i32() + if err != nil { + return ImageSource{}, err + } + switch tag { + case C.IMG_EXTERNAL: + url, err := d.str() + if err != nil { + return ImageSource{}, err + } + return ImageSource{Kind: "external", URL: &url}, nil + case C.IMG_ASSET: + id, err := d.u64() + if err != nil { + return ImageSource{}, err + } + return ImageSource{Kind: "asset", AssetID: &id}, nil + case C.IMG_UNAVAILABLE: + return ImageSource{Kind: "unavailable"}, nil + default: + return ImageSource{}, fmt.Errorf("anydoc: unknown image source tag %d", tag) + } +} + +func (d *decoder) list() (*List, error) { + markerTag, err := d.i32() + if err != nil { + return nil, err + } + start, err := d.u64() + if err != nil { + return nil, err + } + n, err := d.u32() + if err != nil { + return nil, err + } + items := make([]ListItem, 0, d.capFor(n)) + for i := uint32(0); i < n; i++ { + item, err := d.listItem() + if err != nil { + return nil, err + } + items = append(items, item) + } + marker, err := markerName(markerTag) + if err != nil { + return nil, err + } + return &List{Marker: marker, Start: start, Items: items}, nil +} + +func (d *decoder) listItem() (ListItem, error) { + blocks, err := d.blocks() + if err != nil { + return ListItem{}, err + } + checkedTag, err := d.i32() + if err != nil { + return ListItem{}, err + } + var checked *bool + switch checkedTag { + case 0: + f := false + checked = &f + case 1: + t := true + checked = &t + case -1: + checked = nil + default: + return ListItem{}, fmt.Errorf("anydoc: unknown checked tag %d", checkedTag) + } + label, err := d.optStr() + if err != nil { + return ListItem{}, err + } + return ListItem{Blocks: blocks, Checked: checked, MarkerLabel: label}, nil +} + +func (d *decoder) table() (*Table, error) { + headerRows, err := d.u32() + if err != nil { + return nil, err + } + kindTag, err := d.i32() + if err != nil { + return nil, err + } + rows, err := d.u32() + if err != nil { + return nil, err + } + grid := make([][]CellSlot, 0, d.capFor(rows)) + for i := uint32(0); i < rows; i++ { + cols, err := d.u32() + if err != nil { + return nil, err + } + row := make([]CellSlot, 0, d.capFor(cols)) + for j := uint32(0); j < cols; j++ { + slot, err := d.slot() + if err != nil { + return nil, err + } + row = append(row, slot) + } + grid = append(grid, row) + } + kind := "data" + if kindTag == C.TABLE_LAYOUT { + kind = "layout" + } + return &Table{Grid: grid, HeaderRows: headerRows, Kind: kind}, nil +} + +func (d *decoder) slot() (CellSlot, error) { + tag, err := d.i32() + if err != nil { + return CellSlot{}, err + } + switch tag { + case C.SLOT_ORIGIN: + cell, err := d.cell() + if err != nil { + return CellSlot{}, err + } + return CellSlot{Kind: "origin", Cell: &cell}, nil + case C.SLOT_COVERED: + r, err := d.u32() + if err != nil { + return CellSlot{}, err + } + c, err := d.u32() + if err != nil { + return CellSlot{}, err + } + rr := r + cc := c + return CellSlot{Kind: "covered", OriginRow: &rr, OriginCol: &cc}, nil + default: + return CellSlot{}, fmt.Errorf("anydoc: unknown cell slot tag %d", tag) + } +} + +func (d *decoder) cell() (Cell, error) { + colSpan, err := d.u32() + if err != nil { + return Cell{}, err + } + rowSpan, err := d.u32() + if err != nil { + return Cell{}, err + } + blocks, err := d.blocks() + if err != nil { + return Cell{}, err + } + return Cell{Blocks: blocks, ColSpan: colSpan, RowSpan: rowSpan}, nil +} + +func (d *decoder) notes() ([]Note, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Note, 0, d.capFor(n)) + for i := uint32(0); i < n; i++ { + note, err := d.note() + if err != nil { + return nil, err + } + out = append(out, note) + } + return out, nil +} + +func (d *decoder) note() (Note, error) { + id, err := d.str() + if err != nil { + return Note{}, err + } + kindTag, err := d.i32() + if err != nil { + return Note{}, err + } + blocks, err := d.blocks() + if err != nil { + return Note{}, err + } + kind := "footnote" + if kindTag == C.NOTE_ENDNOTE { + kind = "endnote" + } + return Note{ID: id, Kind: kind, Blocks: blocks}, nil +} + +func (d *decoder) assets() ([]Asset, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Asset, 0, d.capFor(n)) + for i := uint32(0); i < n; i++ { + asset, err := d.asset() + if err != nil { + return nil, err + } + out = append(out, asset) + } + return out, nil +} + +func (d *decoder) asset() (Asset, error) { + id, err := d.u64() + if err != nil { + return Asset{}, err + } + mediaType, err := d.str() + if err != nil { + return Asset{}, err + } + originPart, err := d.str() + if err != nil { + return Asset{}, err + } + data, err := d.bytes() + if err != nil { + return Asset{}, err + } + return Asset{ID: id, MediaType: mediaType, OriginPart: originPart, Data: data}, nil +} + +func markerName(tag int32) (string, error) { + switch tag { + case C.MARKER_BULLET: + return "bullet", nil + case C.MARKER_DECIMAL: + return "decimal", nil + case C.MARKER_LOWER_ALPHA: + return "lower_alpha", nil + case C.MARKER_UPPER_ALPHA: + return "upper_alpha", nil + case C.MARKER_LOWER_ROMAN: + return "lower_roman", nil + case C.MARKER_UPPER_ROMAN: + return "upper_roman", nil + } + return "", fmt.Errorf("anydoc: unknown list marker tag %d", tag) +} diff --git a/third_party/anydoc-go/src/asset_links.rs b/third_party/anydoc-go/src/asset_links.rs new file mode 100644 index 00000000..b5247df7 --- /dev/null +++ b/third_party/anydoc-go/src/asset_links.rs @@ -0,0 +1,83 @@ +//! Rewrite embedded images to markdown-linkable sources so anydoc's official +//! GFM serializer will emit `![alt](images/image-N.ext)` in place. +//! +//! `document_to_markdown` drops `ImageSource::Asset` to alt text because +//! Markdown cannot carry bytes. `ImageSource::External` is already rendered +//! as a normal image link. We keep the official serializer and only change +//! how those assets are addressed. + +use anydoc::model::{Block, CellSlot, Document, ImageSource, Inline}; + +pub fn rewrite_asset_images(document: &mut Document) { + let urls: Vec> = document + .assets + .iter() + .enumerate() + .map(|(i, asset)| { + if asset.bytes.is_empty() { + None + } else { + Some(format!("images/image-{}{}", i + 1, extension_for(&asset.media_type))) + } + }) + .collect(); + rewrite_blocks(&mut document.blocks, &urls); + for note in &mut document.notes { + rewrite_blocks(&mut note.blocks, &urls); + } +} + +fn rewrite_blocks(blocks: &mut [Block], urls: &[Option]) { + for block in blocks { + match block { + Block::Heading { content, .. } | Block::Paragraph(content) => { + rewrite_inlines(content, urls); + } + Block::List(list) => { + for item in &mut list.items { + rewrite_blocks(&mut item.blocks, urls); + } + } + Block::Table(table) => { + for row in &mut table.grid { + for slot in row { + if let CellSlot::Origin(cell) = slot { + rewrite_blocks(&mut cell.blocks, urls); + } + } + } + } + Block::BlockQuote(inner) => rewrite_blocks(inner, urls), + Block::CodeBlock { .. } | Block::Rule => {} + } + } +} + +fn rewrite_inlines(inlines: &mut [Inline], urls: &[Option]) { + for inline in inlines { + match inline { + Inline::Image { source, .. } => { + if let ImageSource::Asset(id) = source { + if let Some(Some(url)) = urls.get(id.0) { + *source = ImageSource::External(url.clone()); + } + } + } + Inline::Link { content, .. } => rewrite_inlines(content, urls), + _ => {} + } + } +} + +fn extension_for(media_type: &str) -> &'static str { + match media_type.to_ascii_lowercase().as_str() { + "image/jpeg" | "image/jpg" => ".jpg", + "image/png" => ".png", + "image/gif" => ".gif", + "image/webp" => ".webp", + "image/bmp" => ".bmp", + "image/tiff" => ".tiff", + "image/svg+xml" => ".svg", + _ => ".bin", + } +} diff --git a/third_party/anydoc-go/src/lib.rs b/third_party/anydoc-go/src/lib.rs new file mode 100644 index 00000000..18b787f7 --- /dev/null +++ b/third_party/anydoc-go/src/lib.rs @@ -0,0 +1,620 @@ +//! C ABI surface for the Go bindings. +//! +//! Memory ownership rule: **Rust allocates, C ABI exposes a `_free` for every +//! allocation family, Go releases handles.** There are two allocation +//! families: +//! +//! - strings: `*mut c_char` + length out-params, returned by +//! `anydoc_to_markdown*` and `anydoc_last_error`. Released via +//! `anydoc_string_free`. Length is passed separately so binary-safe. +//! - document buffer: a single `*mut u8` + length, returned by +//! `anydoc_to_document`, holding the flat serialization from [`model`]. +//! Released via `anydoc_buffer_free`. +//! +//! Errors surface as stable non-zero codes (see `ERR_*`); the human-readable +//! message is available through `anydoc_last_error`, thread-local. + +#![allow(clippy::missing_safety_doc)] + +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; +use std::ptr; + +use anydoc::{ConvertError, Format, document_to_markdown}; + +mod asset_links; +mod model; + +// ---- Error codes ----------------------------------------------------------- + +/// Success. +pub const ERR_OK: c_int = 0; +/// `ConvertError::Unsupported`. +pub const ERR_UNSUPPORTED: c_int = 1; +/// `ConvertError::Malformed`. +pub const ERR_MALFORMED: c_int = 2; +/// `ConvertError::Encrypted`. +pub const ERR_ENCRYPTED: c_int = 3; +/// `ConvertError::ResourceLimit`. +pub const ERR_RESOURCE_LIMIT: c_int = 4; +/// `ConvertError::MissingPart`. +pub const ERR_MISSING_PART: c_int = 5; +/// `ConvertError::Io`. +pub const ERR_IO: c_int = 6; +/// PDF was passed to `anydoc_to_document` (unsupported there). +pub const ERR_PDF_NO_MODEL: c_int = 7; +/// Null pointer or invalid argument passed to the ABI. +pub const ERR_INVALID_ARG: c_int = 8; +/// Unknown format name passed to the ABI. +pub const ERR_UNKNOWN_FORMAT: c_int = 9; + +thread_local! { + static LAST_ERROR: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +fn set_last_error(msg: &str) { + let cstring = CString::new(msg).unwrap_or_else(|_| CString::new("").unwrap()); + LAST_ERROR.with(|cell| *cell.borrow_mut() = Some(cstring)); +} + +fn error_code(err: &ConvertError) -> c_int { + match err { + ConvertError::Unsupported(_) => ERR_UNSUPPORTED, + ConvertError::Malformed { .. } => ERR_MALFORMED, + ConvertError::Encrypted => ERR_ENCRYPTED, + ConvertError::ResourceLimit { .. } => ERR_RESOURCE_LIMIT, + ConvertError::MissingPart { .. } => ERR_MISSING_PART, + ConvertError::Io(_) => ERR_IO, + _ => ERR_UNSUPPORTED, + } +} + +fn fail(err: ConvertError) -> c_int { + let code = error_code(&err); + set_last_error(&err.to_string()); + code +} + +/// Run a conversion with panics contained. A panic that escapes an +/// `extern "C"` function aborts the process (Rust 1.81+), so one malformed +/// document would take the whole host process down instead of failing a single +/// conversion. A caught panic surfaces as `Malformed`, the same as any other +/// unusable input. +/// +/// This cannot contain an abort the runtime raises on its own (stack overflow, +/// allocation failure); those still kill the process. +fn guarded( + entry: &str, + parse: impl FnOnce() -> Result, +) -> Result { + // AssertUnwindSafe: the closure only reads the input slice and returns an + // owned result. Nothing observable is left half-written when it panics, + // because the out-params are written by the caller after this returns. + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(parse)) { + Ok(result) => result, + Err(_) => Err(ConvertError::Malformed { + part: None, + detail: format!("panic while converting in {entry}"), + }), + } +} + +// ---- Format enum ----------------------------------------------------------- + +/// C-side format tag. Stable; mirrors the Node/Python lowercase string names +/// via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel. +pub const ANYDOC_FORMAT_NONE: c_int = -1; +pub const ANYDOC_FORMAT_DOC: c_int = 0; +pub const ANYDOC_FORMAT_DOCX: c_int = 1; +pub const ANYDOC_FORMAT_ODT: c_int = 2; +pub const ANYDOC_FORMAT_PDF: c_int = 3; +pub const ANYDOC_FORMAT_PPT: c_int = 4; +pub const ANYDOC_FORMAT_PPTX: c_int = 5; +pub const ANYDOC_FORMAT_RTF: c_int = 6; +pub const ANYDOC_FORMAT_EPUB: c_int = 7; +pub const ANYDOC_FORMAT_XLSX: c_int = 8; +pub const ANYDOC_FORMAT_ODS: c_int = 9; +pub const ANYDOC_FORMAT_ODP: c_int = 10; +pub const ANYDOC_FORMAT_CSV: c_int = 11; + +fn format_to_c(format: Format) -> c_int { + match format { + Format::Doc => ANYDOC_FORMAT_DOC, + Format::Docx => ANYDOC_FORMAT_DOCX, + Format::Odt => ANYDOC_FORMAT_ODT, + Format::Pdf => ANYDOC_FORMAT_PDF, + Format::Ppt => ANYDOC_FORMAT_PPT, + Format::Pptx => ANYDOC_FORMAT_PPTX, + Format::Rtf => ANYDOC_FORMAT_RTF, + Format::Epub => ANYDOC_FORMAT_EPUB, + Format::Excel => ANYDOC_FORMAT_XLSX, + Format::Ods => ANYDOC_FORMAT_ODS, + Format::Odp => ANYDOC_FORMAT_ODP, + Format::Csv => ANYDOC_FORMAT_CSV, + } +} + +fn format_from_c(tag: c_int) -> Option { + Some(match tag { + ANYDOC_FORMAT_DOC => Format::Doc, + ANYDOC_FORMAT_DOCX => Format::Docx, + ANYDOC_FORMAT_ODT => Format::Odt, + ANYDOC_FORMAT_PDF => Format::Pdf, + ANYDOC_FORMAT_PPT => Format::Ppt, + ANYDOC_FORMAT_PPTX => Format::Pptx, + ANYDOC_FORMAT_RTF => Format::Rtf, + ANYDOC_FORMAT_EPUB => Format::Epub, + ANYDOC_FORMAT_XLSX => Format::Excel, + ANYDOC_FORMAT_ODS => Format::Ods, + ANYDOC_FORMAT_ODP => Format::Odp, + ANYDOC_FORMAT_CSV => Format::Csv, + _ => return None, + }) +} + +/// Format names, lowercase, matching the Node/Python convention. +const FORMAT_NAMES: [&str; 12] = + ["doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv"]; + +fn format_name(format: Format) -> &'static str { + match format { + Format::Doc => FORMAT_NAMES[0], + Format::Docx => FORMAT_NAMES[1], + Format::Odt => FORMAT_NAMES[2], + Format::Pdf => FORMAT_NAMES[3], + Format::Ppt => FORMAT_NAMES[4], + Format::Pptx => FORMAT_NAMES[5], + Format::Rtf => FORMAT_NAMES[6], + Format::Epub => FORMAT_NAMES[7], + Format::Excel => FORMAT_NAMES[8], + Format::Ods => FORMAT_NAMES[9], + Format::Odp => FORMAT_NAMES[10], + Format::Csv => FORMAT_NAMES[11], + } +} + +fn parse_format_name(name: &str) -> Option { + FORMAT_NAMES + .iter() + .zip([ + Format::Doc, + Format::Docx, + Format::Odt, + Format::Pdf, + Format::Ppt, + Format::Pptx, + Format::Rtf, + Format::Epub, + Format::Excel, + Format::Ods, + Format::Odp, + Format::Csv, + ]) + .find(|(n, _)| **n == name) + .map(|(_, f)| f) +} + +// ---- String out-param helper ---------------------------------------------- + +/// Box a `String` into a heap allocation the caller owns and must free with +/// `anydoc_string_free`. Writes the byte length (excluding any NUL) to +/// `*out_len`. The buffer is NUL-terminated for safety with C string readers, +/// but binary-safe content relies on `out_len`. +/// +/// Returns a dangling null pointer for empty strings so the Go side can treat +/// a null result as "empty" without needing to free it. Length is still +/// written as 0. +fn string_out(s: String, out_str: *mut *mut c_char, out_len: *mut usize) -> c_int { + if out_str.is_null() { + set_last_error("anydoc: out_str is null"); + return ERR_INVALID_ARG; + } + let len = s.len(); + // SAFETY: we write the length unconditionally; null is the empty sentinel. + if !out_len.is_null() { + unsafe { *out_len = len }; + } + if len == 0 { + // Empty string: return null sentinel, no allocation to free. + unsafe { *out_str = ptr::null_mut() }; + return ERR_OK; + } + let cstring = match CString::new(s.into_bytes()) { + Ok(cs) => cs, + Err(_) => { + // Contains an interior NUL: fall back to lossy replacement so the + // caller still gets something. Markdown shouldn't contain NULs. + set_last_error("anydoc: output contained an interior NUL byte"); + return ERR_INVALID_ARG; + } + }; + let raw = cstring.into_raw(); + // into_raw returns a buffer of capacity len+1 (with the trailing NUL). + unsafe { *out_str = raw }; + ERR_OK +} + +// ---- C ABI functions ------------------------------------------------------- + +/// Detect the format from the content. Writes a format tag to `*out` (one of +/// `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches. +/// +/// Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null +/// pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_bytes( + bytes: *const u8, + len: usize, + out: *mut c_int, +) -> c_int { + if bytes.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_bytes"); + return ERR_INVALID_ARG; + } + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + let tag = Format::from_bytes(slice).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// The format a bare extension names (no leading dot). Writes a format tag to +/// `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_extension( + ext: *const c_char, + out: *mut c_int, +) -> c_int { + if ext.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_extension"); + return ERR_INVALID_ARG; + } + let ext_str = match unsafe { std::ffi::CStr::from_ptr(ext) }.to_str() { + Ok(s) => s.trim_start_matches('.'), + Err(_) => { + set_last_error("anydoc: extension is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = Format::from_extension(ext_str).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// The format a path's extension names. Writes a format tag to `*out`, or +/// `ANYDOC_FORMAT_NONE` when unrecognized. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_path(path: *const c_char, out: *mut c_int) -> c_int { + if path.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_path"); + return ERR_INVALID_ARG; + } + let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: path is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = + Format::from_path(std::path::Path::new(path_str)).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// Convert a document file to Markdown. The format is detected from the file +/// content, with the extension as the fallback for signature-less formats +/// (CSV). +/// +/// On success writes a `*mut c_char` to `*out_str` and its byte length to +/// `*out_len`. The caller must free the string with `anydoc_string_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_markdown( + path: *const c_char, + out_str: *mut *mut c_char, + out_len: *mut usize, +) -> c_int { + if path.is_null() || out_str.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_markdown"); + return ERR_INVALID_ARG; + } + let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: path is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + match guarded("anydoc_to_markdown", || anydoc::to_markdown(path_str)) { + Ok(md) => string_out(md, out_str, out_len), + Err(e) => fail(e), + } +} + +/// Convert an in-memory document to Markdown. `format_tag` is one of the +/// `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the +/// content. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_markdown_bytes( + bytes: *const u8, + len: usize, + format_tag: c_int, + out_str: *mut *mut c_char, + out_len: *mut usize, +) -> c_int { + if bytes.is_null() || out_str.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_markdown_bytes"); + return ERR_INVALID_ARG; + } + let format = if format_tag == ANYDOC_FORMAT_NONE { + None + } else { + match format_from_c(format_tag) { + Some(f) => Some(f), + None => { + set_last_error("anydoc: unknown format tag"); + return ERR_UNKNOWN_FORMAT; + } + } + }; + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + match guarded("anydoc_to_markdown_bytes", || anydoc::to_markdown_bytes(slice, format)) { + Ok(md) => string_out(md, out_str, out_len), + Err(e) => fail(e), + } +} + +/// Convert an in-memory document to Markdown with embedded images rewritten +/// as `![alt](images/image-N.ext)` so they keep their original positions. +/// +/// This is `to_document` + anydoc's official GFM serializer. Asset images are +/// first turned into `ImageSource::External` URLs; the serializer already +/// emits those as ordinary Markdown image links. PDF has no document model +/// and is converted the same way as `anydoc_to_markdown_bytes`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_markdown_with_asset_links( + bytes: *const u8, + len: usize, + format_tag: c_int, + out_str: *mut *mut c_char, + out_len: *mut usize, +) -> c_int { + if bytes.is_null() || out_str.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_markdown_with_asset_links"); + return ERR_INVALID_ARG; + } + let format = if format_tag == ANYDOC_FORMAT_NONE { + None + } else { + match format_from_c(format_tag) { + Some(f) => Some(f), + None => { + set_last_error("anydoc: unknown format tag"); + return ERR_UNKNOWN_FORMAT; + } + } + }; + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + match guarded("anydoc_to_markdown_with_asset_links", || { + markdown_with_asset_links(slice, format) + }) { + Ok(md) => string_out(md, out_str, out_len), + Err(e) => fail(e), + } +} + +fn markdown_with_asset_links( + bytes: &[u8], + format: Option, +) -> Result { + let resolved = format + .or_else(|| Format::from_bytes(bytes)) + .ok_or_else(|| { + ConvertError::Unsupported( + "unrecognized file content: name the format explicitly".into(), + ) + })?; + if resolved == Format::Pdf { + return anydoc::to_markdown_bytes(bytes, resolved); + } + let mut document = anydoc::to_document(bytes, resolved)?; + asset_links::rewrite_asset_images(&mut document); + Ok(document_to_markdown(&document)) +} + +/// Parse an in-memory document into the document model. `format_tag` is one of +/// the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the +/// content. +/// +/// On success writes a heap-allocated byte buffer to `*out_buf` and its length +/// to `*out_len`. The buffer holds the flat serialization (see `model`). The +/// caller must free it with `anydoc_buffer_free`. +/// +/// Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown +/// directly and has no document-model form. Use `anydoc_to_markdown_bytes`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_document( + bytes: *const u8, + len: usize, + format_tag: c_int, + out_buf: *mut *mut u8, + out_len: *mut usize, +) -> c_int { + if bytes.is_null() || out_buf.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_document"); + return ERR_INVALID_ARG; + } + let format = if format_tag == ANYDOC_FORMAT_NONE { + None + } else { + match format_from_c(format_tag) { + Some(f) => Some(f), + None => { + set_last_error("anydoc: unknown format tag"); + return ERR_UNKNOWN_FORMAT; + } + } + }; + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + // Resolve the format up front so we can give the typed PDF error before + // parsing, matching the crate's `to_document` contract. + let resolved = match format { + Some(f) => f, + None => match Format::from_bytes(slice) { + Some(f) => f, + None => { + set_last_error( + "unsupported input: unrecognized file content: name the format explicitly", + ); + return ERR_UNSUPPORTED; + } + }, + }; + if resolved == Format::Pdf { + set_last_error( + "unsupported input: PDF conversion produces Markdown directly and has no document-model form; use to_markdown_bytes", + ); + return ERR_PDF_NO_MODEL; + } + let document = match guarded("anydoc_to_document", || anydoc::to_document(slice, resolved)) { + Ok(d) => d, + Err(e) => return fail(e), + }; + let mut encoder = model::Encoder::new(); + model::write_document(&mut encoder, &document); + let body = encoder.into_vec(); + let len = body.len(); + // Layout: [usize len][usize cap][u8 data..len]. The caller receives a + // pointer to the data area and the length; `anydoc_buffer_free` walks back + // to the header to recover (len, cap) for a sound deallocation. + let header_size = std::mem::size_of::(); + let total = header_size + len; + let layout = std::alloc::Layout::from_size_align(total, std::mem::align_of::()) + .expect("document buffer layout"); + let base = unsafe { std::alloc::alloc(layout) as *mut BufferHeader }; + if base.is_null() { + set_last_error("anydoc: out of memory"); + return ERR_IO; + } + // The capacity we record is the *total* allocation size, so the free path + // can reconstruct a layout of the same size to pass back to `dealloc`. + unsafe { + std::ptr::write(base, BufferHeader { len, cap: total }); + let data = (base as *mut u8).add(header_size); + std::ptr::copy_nonoverlapping(body.as_ptr(), data, len); + *out_buf = data; + if !out_len.is_null() { + *out_len = len; + } + } + // `body` drops here, freeing its own backing allocation. + ERR_OK +} + +/// Header prepended to every document buffer, so `anydoc_buffer_free` can +/// soundly reclaim the allocation without any thread-local state. The data +/// the caller sees starts immediately after this header. +#[repr(C)] +struct BufferHeader { + len: usize, + /// Total allocation size (header + data), used to rebuild the Layout for + /// `dealloc`. + cap: usize, +} + +/// Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`. +/// Null is a no-op (the empty-string sentinel). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_string_free(s: *mut c_char) { + if s.is_null() { + return; + } + // SAFETY: the pointer was produced by `CString::into_raw`; reclaiming it + // with `from_raw` is the matching deallocator. + unsafe { + let _ = CString::from_raw(s); + } +} + +/// Free a document buffer returned by `anydoc_to_document`. Null is a no-op. +/// `len` must be the length written to `*out_len` at allocation time; it is +/// validated against the header for safety. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_buffer_free(buf: *mut u8, len: usize) { + if buf.is_null() { + return; + } + let header_size = std::mem::size_of::(); + // SAFETY: the pointer was produced by `anydoc_to_document` with a + // `BufferHeader` immediately before it. Walk back to read the header and + // rebuild the Layout, then dealloc the whole block. + unsafe { + let base = (buf.sub(header_size)) as *mut BufferHeader; + let header = base.read(); + if header.len != len { + // Length mismatch: caller bug. Leak rather than risk a bad free. + set_last_error("anydoc: anydoc_buffer_free length does not match allocation"); + return; + } + let layout = std::alloc::Layout::from_size_align_unchecked( + header.cap, + std::mem::align_of::(), + ); + std::alloc::dealloc(base as *mut u8, layout); + } +} + +/// Return the human-readable message for the last error on the current +/// thread. The returned `*mut c_char` is freshly allocated and must be freed +/// with `anydoc_string_free`. Returns null when there is no last error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_last_error() -> *mut c_char { + LAST_ERROR + .with(|cell| cell.borrow_mut().take()) + .and_then(|cstring| { + // Re-allocate as a fresh owned C string the caller can free. + let bytes = cstring.into_bytes_with_nul(); + CString::from_vec_with_nul(bytes).ok().map(|cs| cs.into_raw()) + }) + .unwrap_or(ptr::null_mut()) +} + +// ---- Go-side helpers exposed for parity tests ----------------------------- + +/// Return the lowercase name of a format tag, or null for an unknown tag. +/// The returned string must be freed with `anydoc_string_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_name(tag: c_int) -> *mut c_char { + if tag == ANYDOC_FORMAT_NONE { + return ptr::null_mut(); + } + match format_from_c(tag) { + Some(f) => { + let name = format_name(f); + match CString::new(name) { + Ok(cs) => cs.into_raw(), + Err(_) => ptr::null_mut(), + } + } + None => ptr::null_mut(), + } +} + +/// Parse a lowercase format name into a tag. Writes the tag to `*out`, or +/// `ANYDOC_FORMAT_NONE` when the name is unknown. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_name(name: *const c_char, out: *mut c_int) -> c_int { + if name.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_name"); + return ERR_INVALID_ARG; + } + let name_str = match unsafe { std::ffi::CStr::from_ptr(name) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: format name is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = parse_format_name(name_str).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} diff --git a/third_party/anydoc-go/src/model.rs b/third_party/anydoc-go/src/model.rs new file mode 100644 index 00000000..5a97b044 --- /dev/null +++ b/third_party/anydoc-go/src/model.rs @@ -0,0 +1,321 @@ +//! `#[repr(C)]` DTOs and a flat, self-describing serialization of the document +//! model. +//! +//! The document tree is recursive (blocks hold blocks, inlines hold inlines, +//! tables hold cells that hold blocks), so rather than surfacing it through a +//! forest of opaque handles and one `_free` per node type, the FFI serializes +//! the whole `Document` into a single length-prefixed byte buffer. That gives +//! exactly one allocation family for the tree (`anydoc_buffer_free`) and keeps +//! the C ABI tiny. Variant tags are written inline as `c_int`-compatible +//! values, mirroring the Node/Python `kind` string convention. +//! +//! Format: see `write_document` below. Every field is length-prefixed so the +//! Go side can decode without guessing. Strings carry their byte length as a +//! u32; slices carry a u32 count followed by that many elements. Booleans are +//! a single byte (0 or 1). Integers are little-endian fixed width. + +use std::os::raw::c_int; + +use anydoc::model; + +/// Tag values for `Block` variants. Stable across versions; the Go side maps +/// these to the same lowercase string names the Node and Python bindings use. +pub const BLOCK_HEADING: c_int = 0; +pub const BLOCK_PARAGRAPH: c_int = 1; +pub const BLOCK_LIST: c_int = 2; +pub const BLOCK_TABLE: c_int = 3; +pub const BLOCK_QUOTE: c_int = 4; +pub const BLOCK_CODE: c_int = 5; +pub const BLOCK_RULE: c_int = 6; + +pub const INLINE_TEXT: c_int = 0; +pub const INLINE_LINK: c_int = 1; +pub const INLINE_IMAGE: c_int = 2; +pub const INLINE_ANCHOR: c_int = 3; +pub const INLINE_NOTEREF: c_int = 4; +pub const INLINE_LINEBREAK: c_int = 5; + +pub const LINK_EXTERNAL: c_int = 0; +pub const LINK_RELATIVE: c_int = 1; +pub const LINK_ANCHOR: c_int = 2; + +pub const IMG_EXTERNAL: c_int = 0; +pub const IMG_ASSET: c_int = 1; +pub const IMG_UNAVAILABLE: c_int = 2; + +pub const MARKER_BULLET: c_int = 0; +pub const MARKER_DECIMAL: c_int = 1; +pub const MARKER_LOWER_ALPHA: c_int = 2; +pub const MARKER_UPPER_ALPHA: c_int = 3; +pub const MARKER_LOWER_ROMAN: c_int = 4; +pub const MARKER_UPPER_ROMAN: c_int = 5; + +pub const TABLE_DATA: c_int = 0; +pub const TABLE_LAYOUT: c_int = 1; + +pub const SLOT_ORIGIN: c_int = 0; +pub const SLOT_COVERED: c_int = 1; + +pub const NOTE_FOOTNOTE: c_int = 0; +pub const NOTE_ENDNOTE: c_int = 1; + +/// Encoder for the flat document buffer. Little-endian, length-prefixed. +pub struct Encoder { + buf: Vec, +} + +impl Encoder { + pub fn new() -> Self { + Encoder { buf: Vec::with_capacity(8 * 1024) } + } + + pub fn into_vec(self) -> Vec { + self.buf + } + + pub fn u8(&mut self, v: u8) { + self.buf.push(v); + } + + pub fn bool(&mut self, v: bool) { + self.buf.push(if v { 1 } else { 0 }); + } + + pub fn u32(&mut self, v: u32) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + pub fn u64(&mut self, v: u64) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + pub fn i32(&mut self, v: i32) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + /// A length-prefixed UTF-8 string. `None` writes a -1 length sentinel. + pub fn opt_str(&mut self, v: &Option) { + match v { + Some(s) => self.str(s), + None => self.i32(-1), + } + } + + pub fn str(&mut self, s: &str) { + self.u32(s.len() as u32); + self.buf.extend_from_slice(s.as_bytes()); + } + + /// A length-prefixed byte slice. + pub fn bytes(&mut self, b: &[u8]) { + self.u32(b.len() as u32); + self.buf.extend_from_slice(b); + } +} + +pub fn write_document(e: &mut Encoder, doc: &model::Document) { + write_blocks(e, &doc.blocks); + e.u32(doc.notes.len() as u32); + for note in &doc.notes { + write_note(e, note); + } + e.u32(doc.assets.len() as u32); + for asset in &doc.assets { + write_asset(e, asset); + } +} + +fn write_blocks(e: &mut Encoder, blocks: &[model::Block]) { + e.u32(blocks.len() as u32); + for block in blocks { + write_block(e, block); + } +} + +fn write_block(e: &mut Encoder, block: &model::Block) { + match block { + model::Block::Heading { level, anchor, content } => { + e.i32(BLOCK_HEADING); + e.u8(*level); + e.opt_str(anchor); + write_inlines(e, content); + } + model::Block::Paragraph(content) => { + e.i32(BLOCK_PARAGRAPH); + write_inlines(e, content); + } + model::Block::List(list) => { + e.i32(BLOCK_LIST); + write_list(e, list); + } + model::Block::Table(table) => { + e.i32(BLOCK_TABLE); + write_table(e, table); + } + model::Block::BlockQuote(inner) => { + e.i32(BLOCK_QUOTE); + write_blocks(e, inner); + } + model::Block::CodeBlock { lang, text } => { + e.i32(BLOCK_CODE); + e.opt_str(lang); + e.str(text); + } + model::Block::Rule => { + e.i32(BLOCK_RULE); + } + } +} + +fn write_inlines(e: &mut Encoder, inlines: &[model::Inline]) { + e.u32(inlines.len() as u32); + for inline in inlines { + write_inline(e, inline); + } +} + +fn write_inline(e: &mut Encoder, inline: &model::Inline) { + match inline { + model::Inline::Text { text, style } => { + e.i32(INLINE_TEXT); + e.str(text); + write_style(e, *style); + } + model::Inline::Link { content, target } => { + e.i32(INLINE_LINK); + write_inlines(e, content); + write_link_target(e, target); + } + model::Inline::Image { alt, source } => { + e.i32(INLINE_IMAGE); + e.str(alt); + write_image_source(e, source); + } + model::Inline::Anchor(id) => { + e.i32(INLINE_ANCHOR); + e.str(id); + } + model::Inline::NoteRef(id) => { + e.i32(INLINE_NOTEREF); + e.str(id); + } + model::Inline::LineBreak => { + e.i32(INLINE_LINEBREAK); + } + } +} + +fn write_style(e: &mut Encoder, style: model::Style) { + e.bool(style.bold); + e.bool(style.italic); + e.bool(style.strike); + e.bool(style.code); +} + +fn write_link_target(e: &mut Encoder, target: &model::LinkTarget) { + match target { + model::LinkTarget::External(v) => { + e.i32(LINK_EXTERNAL); + e.str(v); + } + model::LinkTarget::Relative(v) => { + e.i32(LINK_RELATIVE); + e.str(v); + } + model::LinkTarget::Anchor(v) => { + e.i32(LINK_ANCHOR); + e.str(v); + } + } +} + +fn write_image_source(e: &mut Encoder, source: &model::ImageSource) { + match source { + model::ImageSource::External(url) => { + e.i32(IMG_EXTERNAL); + e.str(url); + } + model::ImageSource::Asset(id) => { + e.i32(IMG_ASSET); + e.u64(id.0 as u64); + } + model::ImageSource::Unavailable => { + e.i32(IMG_UNAVAILABLE); + } + } +} + +fn write_list(e: &mut Encoder, list: &model::List) { + e.i32(marker_tag(list.marker)); + e.u64(list.start); + e.u32(list.items.len() as u32); + for item in &list.items { + write_list_item(e, item); + } +} + +fn write_list_item(e: &mut Encoder, item: &model::ListItem) { + write_blocks(e, &item.blocks); + match item.checked { + None => e.i32(-1), + Some(false) => e.i32(0), + Some(true) => e.i32(1), + } + e.opt_str(&item.marker_label); +} + +fn write_table(e: &mut Encoder, table: &model::Table) { + e.u32(table.header_rows as u32); + e.i32(if table.kind == model::TableKind::Layout { TABLE_LAYOUT } else { TABLE_DATA }); + e.u32(table.grid.len() as u32); + for row in &table.grid { + e.u32(row.len() as u32); + for slot in row { + write_slot(e, slot); + } + } +} + +fn write_slot(e: &mut Encoder, slot: &model::CellSlot) { + match slot { + model::CellSlot::Origin(cell) => { + e.i32(SLOT_ORIGIN); + write_cell(e, cell); + } + model::CellSlot::Covered { origin_row, origin_col } => { + e.i32(SLOT_COVERED); + e.u32(*origin_row as u32); + e.u32(*origin_col as u32); + } + } +} + +fn write_cell(e: &mut Encoder, cell: &model::Cell) { + e.u32(cell.col_span); + e.u32(cell.row_span); + write_blocks(e, &cell.blocks); +} + +fn write_note(e: &mut Encoder, note: &model::Note) { + e.str(¬e.id); + e.i32(if note.kind == model::NoteKind::Endnote { NOTE_ENDNOTE } else { NOTE_FOOTNOTE }); + write_blocks(e, ¬e.blocks); +} + +fn write_asset(e: &mut Encoder, asset: &model::Asset) { + e.u64(asset.id.0 as u64); + e.str(&asset.media_type); + e.str(&asset.origin_part); + e.bytes(&asset.bytes); +} + +fn marker_tag(marker: model::MarkerKind) -> c_int { + match marker { + model::MarkerKind::Bullet => MARKER_BULLET, + model::MarkerKind::Decimal => MARKER_DECIMAL, + model::MarkerKind::LowerAlpha => MARKER_LOWER_ALPHA, + model::MarkerKind::UpperAlpha => MARKER_UPPER_ALPHA, + model::MarkerKind::LowerRoman => MARKER_LOWER_ROMAN, + model::MarkerKind::UpperRoman => MARKER_UPPER_ROMAN, + } +} diff --git a/third_party/anydoc-go/version.go b/third_party/anydoc-go/version.go new file mode 100644 index 00000000..1d5bfb3c --- /dev/null +++ b/third_party/anydoc-go/version.go @@ -0,0 +1,6 @@ +package anydoc + +// Version is the version of the anydoc-go module, bumped in lockstep with the +// Rust crate version (go/Cargo.toml) and the GitHub release tag. The +// packaged static libraries. +const Version = "0.1.8" diff --git a/website-docs/02-architecture/03-document-pipeline.md b/website-docs/02-architecture/03-document-pipeline.md index 990a7aef..3c2086d1 100644 --- a/website-docs/02-architecture/03-document-pipeline.md +++ b/website-docs/02-architecture/03-document-pipeline.md @@ -282,7 +282,10 @@ Worker 消费 `TypeDocumentProcess` 后按五个规范化阶段推进,每个 4. `resolveDocReader` 返回 `interfaces.DocReader`: - **builtin**:通过 gRPC(`docparser/grpc_parser.go`)或 HTTP(`http_parser.go`)调用 Python **docreader** 服务; - **simple**:Go 原生解析 md/txt/csv/json/图片/音频(`builtin_converter.go`,CSV→Markdown 表格、JSON→递归分割的代码块,图片/音频转占位引用); - - **weknoracloud / mineru / mineru_cloud / paddleocr_vl / paddleocr_vl_cloud**:HTTP 转换器(`engine_registry.go` 注册,按 `mineru_endpoint`、`mineru_api_key`、`paddleocr_vl_endpoint` 等配置判定可用性)。 + - **anydoc**:Go 进程内解析 docx/doc/pptx/ppt/xlsx/xls/odf/rtf/epub/csv/pdf(`anydoc_reader.go`),底层是通过 cgo 链接的 anydoc Rust 库。office 文档的嵌入图按文档模型插回 Markdown 原位;无文字层的扫描件 PDF 在 DocReader 可用时回退到 builtin 整页渲染。仅在带 `anydoc` 构建标签的二进制中可用,其余构建里该引擎在引擎列表中显示为不可用; + - **weknoracloud / mineru / mineru_cloud / paddleocr_vl / paddleocr_vl_cloud**:HTTP 转换器(`engines.go` 注册,按 `mineru_endpoint`、`mineru_api_key`、`paddleocr_vl_endpoint` 等配置判定可用性)。 + +引擎目录集中在 `internal/infrastructure/docparser/engines.go`:每个引擎同时声明元数据(名称、描述、文件类型、可用性探针)与 `NewReader` 工厂,`docparser.NewReader` 按名字分发,未注册的名字(如只存在于 docreader 的 `markitdown`)落到 docreader 客户端。 5. 文件模式:从 `FileService.GetFile(payload.FilePath)` 读回字节填入 `ReadRequest.FileContent`。 **docreader 服务侧**(`docreader/`,Python gRPC):proto 定义 `docreader/proto/docreader.proto`,服务方法 `Read` / `ReadStream`(流式:首帧 meta + 每图一帧,避免大扫描件 PDF 触发 gRPC 消息上限)/ `ListEngines`。内置 parser 覆盖 docx/doc/pdf/md/xlsx/xls/epub/html/htm/mhtml/图片/网页(`WebParser` 处理 URL),并可选注册 `markitdown`(微软 MarkItDown)与 `opendataloader`(PDF 版面分析,需 Java 11+)引擎,Go 侧通过 `ListEngines` 自发现远程引擎。返回统一为 `ReadResult{MarkdownContent, ImageRefs, Metadata, IsAudio, AudioData}` —— **解析产物统一是 Markdown 文本 + 图片字节**,图片持久化由 Go 侧负责。 diff --git a/website-docs/03-features/03-document-parsing.md b/website-docs/03-features/03-document-parsing.md index 4b476842..ad17b276 100644 --- a/website-docs/03-features/03-document-parsing.md +++ b/website-docs/03-features/03-document-parsing.md @@ -400,6 +400,48 @@ Python 依赖(`pyproject.toml` + `uv.lock` 锁定):`grpcio`、`pypdfium2` --- +## 8. anydoc 引擎(Go 进程内解析,不经 docreader) + +`anydoc` 是一个 Go 侧的可选解析引擎:它把 [anydoc](https://github.com/firecrawl/anydoc)(Rust 编写的文档转换库)通过 cgo 链接进 WeKnora 主进程,直接把 office 文档转成 Markdown。与本文其余部分描述的 docreader 不同,它**不经过 Python 服务、不跨进程、也不调用外部二进制**——适合不想部署 docreader 的轻量部署,或对解析延迟敏感的场景。 + +支持的文件类型:`doc`、`docx`、`docm`、`odt`、`rtf`、`ppt`、`pptx`、`pptm`、`odp`、`xls`、`xlsx`、`xlsm`、`ods`、`epub`、`csv`、`pdf`。 + +### 8.1 启用方式 + +解析库是 Rust 静态库,需要 Rust 工具链构建,因此默认不链接:未启用时该引擎在「解析引擎」列表里显示为不可用,并给出提示,其它引擎不受影响。 + +```bash +make build-anydoc # 构建静态库 + 带 anydoc 标签的二进制 +# 等价于: +scripts/build-anydoc-lib.sh && go build -tags anydoc ./cmd/server +``` + +Docker 镜像: + +```bash +docker build -f docker/Dockerfile.app --build-arg WITH_ANYDOC=1 -t weknora-app:anydoc . +``` + +启用后在知识库的解析设置里把对应文件类型指向 `anydoc` 引擎即可。 + +### 8.2 能力边界 + +- **扫描件 PDF**:anydoc 只抽取 PDF 的文字层。没有文字层的扫描件会报「需要 OCR」;若 DocReader(builtin)已连接,AnydocReader 会自动把该文件交给 builtin,按页渲染 JPEG 并标记 `image_source_type=scanned_pdf`,后续仍走 Go 侧 OCR。未连接 DocReader 时转换失败,请改用 `builtin`、`mineru` 或 `paddleocr_vl`。 +- **纵向合并单元格不回填**:docreader 的 `Docx2Parser` 会把纵向合并的值复制到每一行(见 issue #2634),anydoc 只在起始行输出该值,后续行留空。对依赖表格逐行语义的知识库,`builtin` 仍然更稳。 +- **图片位置**:开启图片抽取时,先把文档模型里的嵌入图改写成 `images/image-N.ext` 链接,再交给 anydoc 官方 GFM 序列化,因此图片会留在原段落/表格/列表位置。设置引擎覆盖参数 `anydoc_extract_images=false` 可关闭图片抽取,走更快的纯文本渲染(嵌入图会退化成 alt 文本)。 +- **不处理 URL、图片、音频**:这些仍由 `WebParser`、`SimpleFormatReader` 与 ASR 链路负责。 + +### 8.3 代码位置 + +| 路径 | 作用 | +| --- | --- | +| `internal/infrastructure/docparser/anydoc/` | 适配层:格式映射、可用性判断,以及 cgo / stub 两个后端 | +| `internal/infrastructure/docparser/anydoc_reader.go` | `DocReader` 实现:转换结果与图片引用组装 | +| `internal/infrastructure/docparser/engines.go` | 引擎注册(元数据 + Reader 工厂) | +| `third_party/anydoc-go/` | vendored 的上游 Go 绑定与 C ABI shim(来源与本地改动见该目录 README) | + +--- + ## 附:关键事实速查 - **对外接口**:仅 gRPC,端口 `50051`(`DOCREADER_GRPC_PORT`/`PORT`),RPC:`Read` / `ReadStream` / `ListEngines` + 标准 Health 服务。