feat(思维导图助手)：优化打包
parent
5f4baba012
commit
846739cf77
|
|
@ -0,0 +1,20 @@
|
|||
.git
|
||||
.idea
|
||||
venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.log
|
||||
*.tar
|
||||
*.tar.gz
|
||||
*.zip
|
||||
output
|
||||
outputs
|
||||
tmp
|
||||
.cache
|
||||
node_modules
|
||||
web_ui/node_modules
|
||||
web_ui/dist
|
||||
mineru/cli/static/web
|
||||
test_files
|
||||
tests
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
FROM crpi-vofi3w62lkohhxsp.cn-shanghai.personal.cr.aliyuncs.com/opendatalab-mineru/corex:4.4.0_torch2.7.1_vllm0.11.2_py3.10
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV MINERU_MODEL_SOURCE=local
|
||||
ENV MINERU_MODEL_DIR=/models
|
||||
ENV MINERU_TOOLS_CONFIG_JSON=/root/mineru.json
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
fonts-noto-core \
|
||||
fonts-noto-cjk \
|
||||
fontconfig \
|
||||
libgl1-mesa-glx \
|
||||
&& fc-cache -fv \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip install 'mineru[core]>=2.7.4' --no-deps -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip install \
|
||||
"numpy==1.26.4" \
|
||||
"opencv-python==4.11.0.86" \
|
||||
"pydantic>=2.12,<3" \
|
||||
"modelscope" \
|
||||
"magic-pdf" \
|
||||
boto3 \
|
||||
"pdfminer.six>=20251230" \
|
||||
pypdfium2 \
|
||||
pypdf \
|
||||
reportlab \
|
||||
pdftext \
|
||||
huggingface-hub \
|
||||
json-repair \
|
||||
fast-langdetect \
|
||||
scikit-image \
|
||||
openai \
|
||||
beautifulsoup4 \
|
||||
magika \
|
||||
mineru-vl-utils \
|
||||
qwen-vl-utils \
|
||||
matplotlib \
|
||||
ultralytics \
|
||||
doclayout_yolo==0.0.4 \
|
||||
dill \
|
||||
PyYAML \
|
||||
ftfy \
|
||||
shapely \
|
||||
pyclipper \
|
||||
omegaconf \
|
||||
onnxruntime \
|
||||
python-pptx \
|
||||
python-docx \
|
||||
openpyxl \
|
||||
xlrd \
|
||||
xlwt \
|
||||
pylatexenc \
|
||||
mammoth \
|
||||
-i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python3 -m pip cache purge
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
COPY docker/entrypoint.mineru-api.sh /usr/local/bin/entrypoint.mineru-api.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.mineru-api.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.mineru-api.sh"]
|
||||
CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_DIR=${MINERU_MODEL_DIR:-/models}
|
||||
PIPELINE_MODEL_DIR=${MINERU_PIPELINE_MODEL_DIR:-${MODEL_DIR}/pipeline}
|
||||
VLM_MODEL_DIR=${MINERU_VLM_MODEL_DIR:-${MODEL_DIR}/vlm}
|
||||
CONFIG_PATH=${MINERU_TOOLS_CONFIG_JSON:-/root/mineru.json}
|
||||
|
||||
mkdir -p "$PIPELINE_MODEL_DIR" "$VLM_MODEL_DIR" "$(dirname "$CONFIG_PATH")"
|
||||
|
||||
export CONFIG_PATH PIPELINE_MODEL_DIR VLM_MODEL_DIR
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
config_path = Path(os.environ["CONFIG_PATH"])
|
||||
config = {
|
||||
"bucket_info": {"[default]": [None, None, None]},
|
||||
"latex-delimiter-config": {
|
||||
"display": {"left": "$$", "right": "$$"},
|
||||
"inline": {"left": "$", "right": "$"},
|
||||
},
|
||||
"llm-aided-config": {
|
||||
"title_aided": {
|
||||
"api_key": "",
|
||||
"base_url": "",
|
||||
"model": "",
|
||||
"enable_thinking": False,
|
||||
"enable": False,
|
||||
}
|
||||
},
|
||||
"models-dir": {
|
||||
"pipeline": os.environ["PIPELINE_MODEL_DIR"],
|
||||
"vlm": os.environ["VLM_MODEL_DIR"],
|
||||
},
|
||||
"config_version": "1.3.1",
|
||||
}
|
||||
config_path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
PY
|
||||
|
||||
export MINERU_TOOLS_CONFIG_JSON="$CONFIG_PATH"
|
||||
export MINERU_MODEL_SOURCE=${MINERU_MODEL_SOURCE:-local}
|
||||
export PYTHONPATH=${PYTHONPATH:-/app}
|
||||
|
||||
exec "$@"
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
FROM pytorch/pytorch:2.7.1-cuda12.6-cudnn9-runtime
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV MINERU_MODEL_SOURCE=local
|
||||
ENV MINERU_MODEL_DIR=/models
|
||||
ENV MINERU_TOOLS_CONFIG_JSON=/root/mineru.json
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
fonts-noto-core \
|
||||
fonts-noto-cjk \
|
||||
fontconfig \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
git \
|
||||
&& fc-cache -fv \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN ln -sf "$(command -v python)" /usr/local/bin/python3 && \
|
||||
python -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python -m pip install \
|
||||
'mineru[core]>=2.7.4' \
|
||||
'vllm==0.11.2' \
|
||||
'pydantic>=2.12,<3' \
|
||||
modelscope \
|
||||
magic-pdf \
|
||||
boto3 \
|
||||
'pdfminer.six>=20251230' \
|
||||
pypdfium2 \
|
||||
pypdf \
|
||||
reportlab \
|
||||
pdftext \
|
||||
huggingface-hub \
|
||||
json-repair \
|
||||
fast-langdetect \
|
||||
scikit-image \
|
||||
openai \
|
||||
beautifulsoup4 \
|
||||
magika \
|
||||
mineru-vl-utils \
|
||||
qwen-vl-utils \
|
||||
matplotlib \
|
||||
ultralytics \
|
||||
doclayout_yolo==0.0.4 \
|
||||
dill \
|
||||
PyYAML \
|
||||
ftfy \
|
||||
shapely \
|
||||
pyclipper \
|
||||
omegaconf \
|
||||
onnxruntime-gpu \
|
||||
python-pptx \
|
||||
python-docx \
|
||||
openpyxl \
|
||||
xlrd \
|
||||
xlwt \
|
||||
pylatexenc \
|
||||
mammoth \
|
||||
-i https://mirrors.aliyun.com/pypi/simple && \
|
||||
python -m pip cache purge
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
COPY docker/entrypoint.mineru-api.sh /usr/local/bin/entrypoint.mineru-api.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.mineru-api.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.mineru-api.sh"]
|
||||
CMD ["mineru-api", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
|
@ -2,15 +2,29 @@
|
|||
set -euo pipefail
|
||||
|
||||
VERSION=${VERSION:-$(date +%Y%m%d%H%M%S)}
|
||||
API_FLAVOR=${API_FLAVOR:-corex}
|
||||
ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
echo "Building UnisMindMap API image: unis-mindmap-api:${VERSION}"
|
||||
sudo docker build -f docker/china/Dockerfile \
|
||||
case "$API_FLAVOR" in
|
||||
corex)
|
||||
API_DOCKERFILE=docker/china/Dockerfile.corex
|
||||
;;
|
||||
cuda)
|
||||
API_DOCKERFILE=docker/global/Dockerfile.cuda
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported API_FLAVOR: $API_FLAVOR. Use corex or cuda." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Building UnisMindMap API image (${API_FLAVOR}): unis-mindmap-api:${VERSION}"
|
||||
sudo docker build -f "$API_DOCKERFILE" \
|
||||
-t unis-mindmap-api:${VERSION} \
|
||||
-t unis-mindmap-api:${API_FLAVOR} \
|
||||
-t unis-mindmap-api:latest \
|
||||
-t mineru:test \
|
||||
.
|
||||
|
||||
echo "Building UnisMindMap Web image: unis-mindmap-web:${VERSION}"
|
||||
|
|
|
|||
|
|
@ -7,14 +7,22 @@ WEB_NAME=${WEB_NAME:-mindmap-web}
|
|||
WEB_PORT=${WEB_PORT:-9898}
|
||||
MINERU_API_URL=${MINERU_API_URL:-http://${API_NAME}:8000}
|
||||
MINERU_GPU_MEMORY_UTILIZATION=${MINERU_GPU_MEMORY_UTILIZATION:-0.5}
|
||||
MINERU_MODEL_DIR=${MINERU_MODEL_DIR:-/data/unis/mineru-models}
|
||||
MINERU_PYTHONPATH=${MINERU_PYTHONPATH:-/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages}
|
||||
|
||||
sudo docker network create "$NETWORK" >/dev/null 2>&1 || true
|
||||
sudo docker rm -f "$API_NAME" "$WEB_NAME" >/dev/null 2>&1 || true
|
||||
sudo mkdir -p "$MINERU_MODEL_DIR"/pipeline "$MINERU_MODEL_DIR"/vlm
|
||||
sudo chmod -R 777 "$MINERU_MODEL_DIR"
|
||||
|
||||
sudo docker run -d --name "$API_NAME" --network "$NETWORK" \
|
||||
--privileged --ipc=host --shm-size=32g \
|
||||
-e MINERU_MODEL_SOURCE=local \
|
||||
-e PYTHONPATH=/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages \
|
||||
-v "${MINERU_MODEL_DIR}:/models" \
|
||||
-e MINERU_MODEL_SOURCE="${MINERU_MODEL_SOURCE:-local}" \
|
||||
-e MINERU_MODEL_DIR=/models \
|
||||
-e MINERU_PIPELINE_MODEL_DIR=/models/pipeline \
|
||||
-e MINERU_VLM_MODEL_DIR=/models/vlm \
|
||||
-e PYTHONPATH="$MINERU_PYTHONPATH" \
|
||||
-e MINERU_GPU_MEMORY_UTILIZATION="$MINERU_GPU_MEMORY_UTILIZATION" \
|
||||
unis-mindmap-api:latest \
|
||||
mineru-api --host 0.0.0.0 --port 8000
|
||||
|
|
|
|||
Loading…
Reference in New Issue