UnisKB/deploy_artifacts/download-models.sh

77 lines
2.3 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
IMAGE=${MINDMAP_API_IMAGE:-unis-mindmap-api:latest}
SOURCE=${SOURCE:-modelscope}
MODEL_TYPE=${MODEL_TYPE:-all}
MINERU_MODEL_DIR=${MINERU_MODEL_DIR:-/data/unis/mineru-models}
sudo mkdir -p "$MINERU_MODEL_DIR"/pipeline "$MINERU_MODEL_DIR"/vlm
sudo chmod -R 777 "$MINERU_MODEL_DIR"
sudo docker run --rm -i \
-e SOURCE="$SOURCE" \
-e MODEL_TYPE="$MODEL_TYPE" \
-v "${MINERU_MODEL_DIR}:/models" \
"$IMAGE" \
python3 - <<'PY'
import os
from mineru.utils.enum_class import ModelPath
source = os.environ.get("SOURCE", "modelscope")
model_type = os.environ.get("MODEL_TYPE", "all")
if source == "modelscope":
from modelscope import snapshot_download
pipeline_repo = ModelPath.pipeline_root_modelscope
vlm_repo = ModelPath.vlm_root_modelscope
elif source == "huggingface":
from huggingface_hub import snapshot_download
pipeline_repo = ModelPath.pipeline_root_hf
vlm_repo = ModelPath.vlm_root_hf
else:
raise SystemExit(f"Unsupported SOURCE={source}; use modelscope or huggingface")
pipeline_dir = "/models/pipeline"
vlm_dir = "/models/vlm"
pipeline_paths = [
ModelPath.doclayout_yolo,
ModelPath.yolo_v8_mfd,
ModelPath.unimernet_small,
ModelPath.pytorch_paddle,
ModelPath.layout_reader,
ModelPath.slanet_plus,
ModelPath.unet_structure,
ModelPath.paddle_table_cls,
ModelPath.paddle_orientation_classification,
ModelPath.pp_formulanet_plus_m,
]
def download(repo, local_dir, allow_patterns=None):
kwargs = {"local_dir": local_dir}
if allow_patterns:
kwargs["allow_patterns"] = allow_patterns
try:
return snapshot_download(repo, **kwargs)
except TypeError:
kwargs.pop("local_dir")
return snapshot_download(repo, **kwargs)
if model_type in ("pipeline", "all"):
patterns = []
for path in pipeline_paths:
rel = path.strip("/")
patterns.extend([rel, f"{rel}/*"])
print(f"Downloading pipeline models from {source} to {pipeline_dir}")
download(pipeline_repo, pipeline_dir, patterns)
if model_type in ("vlm", "all"):
print(f"Downloading VLM models from {source} to {vlm_dir}")
download(vlm_repo, vlm_dir)
if model_type not in ("pipeline", "vlm", "all"):
raise SystemExit(f"Unsupported MODEL_TYPE={model_type}; use pipeline, vlm, or all")
print("Model download finished.")
PY