77 lines
2.3 KiB
Bash
77 lines
2.3 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
IMAGE=${MINDMAP_API_IMAGE:-unis-mindmap-api:latest}
|
|
SOURCE=${SOURCE:-modelscope}
|
|
MODEL_TYPE=${MODEL_TYPE:-all}
|
|
MINERU_MODEL_DIR=${MINERU_MODEL_DIR:-/data/unis/mineru-models}
|
|
|
|
sudo mkdir -p "$MINERU_MODEL_DIR"/pipeline "$MINERU_MODEL_DIR"/vlm
|
|
sudo chmod -R 777 "$MINERU_MODEL_DIR"
|
|
|
|
sudo docker run --rm -i \
|
|
-e SOURCE="$SOURCE" \
|
|
-e MODEL_TYPE="$MODEL_TYPE" \
|
|
-v "${MINERU_MODEL_DIR}:/models" \
|
|
"$IMAGE" \
|
|
python3 - <<'PY'
|
|
import os
|
|
from mineru.utils.enum_class import ModelPath
|
|
|
|
source = os.environ.get("SOURCE", "modelscope")
|
|
model_type = os.environ.get("MODEL_TYPE", "all")
|
|
|
|
if source == "modelscope":
|
|
from modelscope import snapshot_download
|
|
pipeline_repo = ModelPath.pipeline_root_modelscope
|
|
vlm_repo = ModelPath.vlm_root_modelscope
|
|
elif source == "huggingface":
|
|
from huggingface_hub import snapshot_download
|
|
pipeline_repo = ModelPath.pipeline_root_hf
|
|
vlm_repo = ModelPath.vlm_root_hf
|
|
else:
|
|
raise SystemExit(f"Unsupported SOURCE={source}; use modelscope or huggingface")
|
|
|
|
pipeline_dir = "/models/pipeline"
|
|
vlm_dir = "/models/vlm"
|
|
pipeline_paths = [
|
|
ModelPath.doclayout_yolo,
|
|
ModelPath.yolo_v8_mfd,
|
|
ModelPath.unimernet_small,
|
|
ModelPath.pytorch_paddle,
|
|
ModelPath.layout_reader,
|
|
ModelPath.slanet_plus,
|
|
ModelPath.unet_structure,
|
|
ModelPath.paddle_table_cls,
|
|
ModelPath.paddle_orientation_classification,
|
|
ModelPath.pp_formulanet_plus_m,
|
|
]
|
|
|
|
def download(repo, local_dir, allow_patterns=None):
|
|
kwargs = {"local_dir": local_dir}
|
|
if allow_patterns:
|
|
kwargs["allow_patterns"] = allow_patterns
|
|
try:
|
|
return snapshot_download(repo, **kwargs)
|
|
except TypeError:
|
|
kwargs.pop("local_dir")
|
|
return snapshot_download(repo, **kwargs)
|
|
|
|
if model_type in ("pipeline", "all"):
|
|
patterns = []
|
|
for path in pipeline_paths:
|
|
rel = path.strip("/")
|
|
patterns.extend([rel, f"{rel}/*"])
|
|
print(f"Downloading pipeline models from {source} to {pipeline_dir}")
|
|
download(pipeline_repo, pipeline_dir, patterns)
|
|
|
|
if model_type in ("vlm", "all"):
|
|
print(f"Downloading VLM models from {source} to {vlm_dir}")
|
|
download(vlm_repo, vlm_dir)
|
|
|
|
if model_type not in ("pipeline", "vlm", "all"):
|
|
raise SystemExit(f"Unsupported MODEL_TYPE={model_type}; use pipeline, vlm, or all")
|
|
|
|
print("Model download finished.")
|
|
PY
|