From 846739cf77343885c0a48469a359ec8675c20ca9 Mon Sep 17 00:00:00 2001 From: panyy Date: Tue, 7 Jul 2026 17:31:59 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E6=80=9D=E7=BB=B4=E5=AF=BC=E5=9B=BE?= =?UTF-8?q?=E5=8A=A9=E6=89=8B)=EF=BC=9A=E4=BC=98=E5=8C=96=E6=89=93?= =?UTF-8?q?=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 20 ++++++++++ docker/china/Dockerfile.corex | 65 ++++++++++++++++++++++++++++++++ docker/entrypoint.mineru-api.sh | 47 +++++++++++++++++++++++ docker/global/Dockerfile.cuda | 67 +++++++++++++++++++++++++++++++++ scripts/build-images.sh | 20 ++++++++-- scripts/start-mindmap.sh | 12 +++++- 6 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/china/Dockerfile.corex create mode 100644 docker/entrypoint.mineru-api.sh create mode 100644 docker/global/Dockerfile.cuda diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e238c85 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.idea +venv +__pycache__ +*.pyc +*.pyo +*.log +*.tar +*.tar.gz +*.zip +output +outputs +tmp +.cache +node_modules +web_ui/node_modules +web_ui/dist +mineru/cli/static/web +test_files +tests diff --git a/docker/china/Dockerfile.corex b/docker/china/Dockerfile.corex new file mode 100644 index 0000000..49249e8 --- /dev/null +++ b/docker/china/Dockerfile.corex @@ -0,0 +1,65 @@ +FROM crpi-vofi3w62lkohhxsp.cn-shanghai.personal.cr.aliyuncs.com/opendatalab-mineru/corex:4.4.0_torch2.7.1_vllm0.11.2_py3.10 + +ENV DEBIAN_FRONTEND=noninteractive +ENV MINERU_MODEL_SOURCE=local +ENV MINERU_MODEL_DIR=/models +ENV MINERU_TOOLS_CONFIG_JSON=/root/mineru.json + +RUN apt-get update && apt-get install -y \ + fonts-noto-core \ + fonts-noto-cjk \ + fontconfig \ + libgl1-mesa-glx \ + && fc-cache -fv \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \ + python3 -m pip install 'mineru[core]>=2.7.4' --no-deps -i https://mirrors.aliyun.com/pypi/simple && \ + python3 -m pip install \ + "numpy==1.26.4" \ + "opencv-python==4.11.0.86" \ + "pydantic>=2.12,<3" \ + "modelscope" \ + "magic-pdf" \ + boto3 \ + "pdfminer.six>=20251230" \ + pypdfium2 \ + pypdf \ + reportlab \ + pdftext \ + huggingface-hub \ + json-repair \ + fast-langdetect \ + scikit-image \ + openai \ + beautifulsoup4 \ + magika \ + mineru-vl-utils \ + qwen-vl-utils \ + matplotlib \ + ultralytics \ + doclayout_yolo==0.0.4 \ + dill \ + PyYAML \ + ftfy \ + shapely \ + pyclipper \ + omegaconf \ + onnxruntime \ + python-pptx \ + python-docx \ + openpyxl \ + xlrd \ + xlwt \ + pylatexenc \ + mammoth \ + -i https://mirrors.aliyun.com/pypi/simple && \ + python3 -m pip cache purge + +WORKDIR /app +COPY . . +COPY docker/entrypoint.mineru-api.sh /usr/local/bin/entrypoint.mineru-api.sh +RUN chmod +x /usr/local/bin/entrypoint.mineru-api.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.mineru-api.sh"] +CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/entrypoint.mineru-api.sh b/docker/entrypoint.mineru-api.sh new file mode 100644 index 0000000..76e05a6 --- /dev/null +++ b/docker/entrypoint.mineru-api.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODEL_DIR=${MINERU_MODEL_DIR:-/models} +PIPELINE_MODEL_DIR=${MINERU_PIPELINE_MODEL_DIR:-${MODEL_DIR}/pipeline} +VLM_MODEL_DIR=${MINERU_VLM_MODEL_DIR:-${MODEL_DIR}/vlm} +CONFIG_PATH=${MINERU_TOOLS_CONFIG_JSON:-/root/mineru.json} + +mkdir -p "$PIPELINE_MODEL_DIR" "$VLM_MODEL_DIR" "$(dirname "$CONFIG_PATH")" + +export CONFIG_PATH PIPELINE_MODEL_DIR VLM_MODEL_DIR + +python3 - <<'PY' +import json +import os +from pathlib import Path + +config_path = Path(os.environ["CONFIG_PATH"]) +config = { + "bucket_info": {"[default]": [None, None, None]}, + "latex-delimiter-config": { + "display": {"left": "$$", "right": "$$"}, + "inline": {"left": "$", "right": "$"}, + }, + "llm-aided-config": { + "title_aided": { + "api_key": "", + "base_url": "", + "model": "", + "enable_thinking": False, + "enable": False, + } + }, + "models-dir": { + "pipeline": os.environ["PIPELINE_MODEL_DIR"], + "vlm": os.environ["VLM_MODEL_DIR"], + }, + "config_version": "1.3.1", +} +config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8") +PY + +export MINERU_TOOLS_CONFIG_JSON="$CONFIG_PATH" +export MINERU_MODEL_SOURCE=${MINERU_MODEL_SOURCE:-local} +export PYTHONPATH=${PYTHONPATH:-/app} + +exec "$@" diff --git a/docker/global/Dockerfile.cuda b/docker/global/Dockerfile.cuda new file mode 100644 index 0000000..de62db7 --- /dev/null +++ b/docker/global/Dockerfile.cuda @@ -0,0 +1,67 @@ +FROM pytorch/pytorch:2.7.1-cuda12.6-cudnn9-runtime + +ENV DEBIAN_FRONTEND=noninteractive +ENV MINERU_MODEL_SOURCE=local +ENV MINERU_MODEL_DIR=/models +ENV MINERU_TOOLS_CONFIG_JSON=/root/mineru.json + +RUN apt-get update && apt-get install -y \ + fonts-noto-core \ + fonts-noto-cjk \ + fontconfig \ + libgl1 \ + libglib2.0-0 \ + git \ + && fc-cache -fv \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN ln -sf "$(command -v python)" /usr/local/bin/python3 && \ + python -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \ + python -m pip install \ + 'mineru[core]>=2.7.4' \ + 'vllm==0.11.2' \ + 'pydantic>=2.12,<3' \ + modelscope \ + magic-pdf \ + boto3 \ + 'pdfminer.six>=20251230' \ + pypdfium2 \ + pypdf \ + reportlab \ + pdftext \ + huggingface-hub \ + json-repair \ + fast-langdetect \ + scikit-image \ + openai \ + beautifulsoup4 \ + magika \ + mineru-vl-utils \ + qwen-vl-utils \ + matplotlib \ + ultralytics \ + doclayout_yolo==0.0.4 \ + dill \ + PyYAML \ + ftfy \ + shapely \ + pyclipper \ + omegaconf \ + onnxruntime-gpu \ + python-pptx \ + python-docx \ + openpyxl \ + xlrd \ + xlwt \ + pylatexenc \ + mammoth \ + -i https://mirrors.aliyun.com/pypi/simple && \ + python -m pip cache purge + +WORKDIR /app +COPY . . +COPY docker/entrypoint.mineru-api.sh /usr/local/bin/entrypoint.mineru-api.sh +RUN chmod +x /usr/local/bin/entrypoint.mineru-api.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.mineru-api.sh"] +CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"] diff --git a/scripts/build-images.sh b/scripts/build-images.sh index 0445beb..79ba276 100644 --- a/scripts/build-images.sh +++ b/scripts/build-images.sh @@ -2,15 +2,29 @@ set -euo pipefail VERSION=${VERSION:-$(date +%Y%m%d%H%M%S)} +API_FLAVOR=${API_FLAVOR:-corex} ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd) cd "$ROOT_DIR" -echo "Building UnisMindMap API image: unis-mindmap-api:${VERSION}" -sudo docker build -f docker/china/Dockerfile \ +case "$API_FLAVOR" in + corex) + API_DOCKERFILE=docker/china/Dockerfile.corex + ;; + cuda) + API_DOCKERFILE=docker/global/Dockerfile.cuda + ;; + *) + echo "Unsupported API_FLAVOR: $API_FLAVOR. Use corex or cuda." >&2 + exit 1 + ;; +esac + +echo "Building UnisMindMap API image (${API_FLAVOR}): unis-mindmap-api:${VERSION}" +sudo docker build -f "$API_DOCKERFILE" \ -t unis-mindmap-api:${VERSION} \ + -t unis-mindmap-api:${API_FLAVOR} \ -t unis-mindmap-api:latest \ - -t mineru:test \ . echo "Building UnisMindMap Web image: unis-mindmap-web:${VERSION}" diff --git a/scripts/start-mindmap.sh b/scripts/start-mindmap.sh index 2c37224..fce9b90 100644 --- a/scripts/start-mindmap.sh +++ b/scripts/start-mindmap.sh @@ -7,14 +7,22 @@ WEB_NAME=${WEB_NAME:-mindmap-web} WEB_PORT=${WEB_PORT:-9898} MINERU_API_URL=${MINERU_API_URL:-http://${API_NAME}:8000} MINERU_GPU_MEMORY_UTILIZATION=${MINERU_GPU_MEMORY_UTILIZATION:-0.5} +MINERU_MODEL_DIR=${MINERU_MODEL_DIR:-/data/unis/mineru-models} +MINERU_PYTHONPATH=${MINERU_PYTHONPATH:-/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages} sudo docker network create "$NETWORK" >/dev/null 2>&1 || true sudo docker rm -f "$API_NAME" "$WEB_NAME" >/dev/null 2>&1 || true +sudo mkdir -p "$MINERU_MODEL_DIR"/pipeline "$MINERU_MODEL_DIR"/vlm +sudo chmod -R 777 "$MINERU_MODEL_DIR" sudo docker run -d --name "$API_NAME" --network "$NETWORK" \ --privileged --ipc=host --shm-size=32g \ - -e MINERU_MODEL_SOURCE=local \ - -e PYTHONPATH=/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages \ + -v "${MINERU_MODEL_DIR}:/models" \ + -e MINERU_MODEL_SOURCE="${MINERU_MODEL_SOURCE:-local}" \ + -e MINERU_MODEL_DIR=/models \ + -e MINERU_PIPELINE_MODEL_DIR=/models/pipeline \ + -e MINERU_VLM_MODEL_DIR=/models/vlm \ + -e PYTHONPATH="$MINERU_PYTHONPATH" \ -e MINERU_GPU_MEMORY_UTILIZATION="$MINERU_GPU_MEMORY_UTILIZATION" \ unis-mindmap-api:latest \ mineru-api --host 0.0.0.0 --port 8000