367 lines
17 KiB
Bash
Executable File
367 lines
17 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ============================================================
|
||
# Nex Animation Studio · 服务器部署与运行(单机生产模式)
|
||
#
|
||
# ./scripts/server.sh install 装依赖 → 构建前端 → 建表 → 导入模型配置与管理员
|
||
# ./scripts/server.sh build 只重新构建前端(改了页面时用)
|
||
# ./scripts/server.sh start 启动(前端产物由后端一并托管,单进程)
|
||
# ./scripts/server.sh stop 停止
|
||
# ./scripts/server.sh restart 重启
|
||
# ./scripts/server.sh status 端口 / 健康 / 默认模型 / ComfyUI / 数据边界
|
||
# ./scripts/server.sh verify 打真实 HTTP 跑一遍 12 节点全流程自检
|
||
# ./scripts/server.sh logs 跟踪日志
|
||
# ./scripts/server.sh systemd 打印 systemd 单元(交给 root 安装)
|
||
#
|
||
# 选项:--port N --host ADDR --no-asr --skip-deps --skip-build --db URL
|
||
#
|
||
# 与 scripts/dev.sh 的分工:dev.sh 面向开发(Vite 热更新 + uvicorn --reload),
|
||
# server.sh 面向部署(构建产物 + 不重载 + 首装数据)。两者共用 .local/run 与
|
||
# .local/logs,但 pid 名不同(server vs backend/frontend),可以并存,只要端口错开。
|
||
#
|
||
# 三个必须写在这里的为什么:
|
||
# 1) 只起一个 uvicorn 进程,绝不加 --workers。TASK_EXECUTOR=inline 时调度守护线程
|
||
# 跑在进程内,多 worker 就是 N 份守护线程同时抢同一批任务、重复派发;
|
||
# 要横向扩展请切 TASK_EXECUTOR=celery 并另起 worker(见 docker-compose.yml)。
|
||
# 2) 前端产物由后端托管(SERVE_FRONTEND=on),所以对外只有一个端口,
|
||
# 反向代理只需转发一处,/api、/media、/docs 与页面同源,不再有 CORS 与代理漂移。
|
||
# 3) 首装顺序是「建表 + 灌数据 → 再启动服务」。先启动也能跑,但应用首启会自己
|
||
# 生成一把模型主密钥、把 mock 播成默认,导入时就得再纠正一次默认位。
|
||
# ============================================================
|
||
set -euo pipefail
|
||
|
||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
RUN_DIR="$ROOT/.local/run"
|
||
LOG_DIR="$ROOT/.local/logs"
|
||
PYBIN="$ROOT/.venv/bin/python"
|
||
NAME=server
|
||
HOST="${NEX_HOST:-0.0.0.0}"
|
||
PORT="${PORT:-8000}"
|
||
ASR_MODEL="$ROOT/models/asr/ggml-small.bin"
|
||
ASR_URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"
|
||
WANT_ASR=1
|
||
SKIP_DEPS=0
|
||
SKIP_BUILD=0
|
||
EXPLICIT_DB="${DATABASE_URL:-}"
|
||
|
||
if [ -t 1 ]; then C_G=$'\033[32m'; C_Y=$'\033[33m'; C_R=$'\033[31m'; C_B=$'\033[36m'; C_D=$'\033[2m'; C_0=$'\033[0m'
|
||
else C_G=""; C_Y=""; C_R=""; C_B=""; C_D=""; C_0=""; fi
|
||
ok() { printf ' %sok%s %s\n' "$C_G" "$C_0" "$*"; }
|
||
info() { printf ' %s\n' "$*"; }
|
||
warn() { printf ' %s!%s %s\n' "$C_Y" "$C_0" "$*"; }
|
||
die() { printf ' %sXX%s %s\n' "$C_R" "$C_0" "$*" >&2; exit 1; }
|
||
head2() { printf '\n%s%s%s\n' "$C_B" "$*" "$C_0"; }
|
||
|
||
pid_of() { [ -f "$RUN_DIR/$NAME.pid" ] && cat "$RUN_DIR/$NAME.pid" 2>/dev/null || true; }
|
||
alive() { local p="${1:-}"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; }
|
||
port_pids() { lsof -nP -iTCP:"$1" -sTCP:LISTEN -t 2>/dev/null | tr '\n' ' ' | sed 's/[[:space:]]*$//' || true; }
|
||
env_value() { [ -f "$ROOT/.env" ] && sed -n "s/^[[:space:]]*$1=//p" "$ROOT/.env" | tail -1 | tr -d '\r"' || true; }
|
||
|
||
# 独立会话里起进程(同 dev.sh:`cmd &` 会被启动它的终端一起带走,SSH 断开即挂)
|
||
spawn() {
|
||
mkdir -p "$RUN_DIR" "$LOG_DIR"
|
||
local pid
|
||
pid="$("$PYBIN" - "$LOG_DIR/$NAME.log" "$ROOT" "$@" <<'PY'
|
||
import subprocess, sys
|
||
log_path, cwd, cmd = sys.argv[1], sys.argv[2], sys.argv[3:]
|
||
log = open(log_path, "ab", buffering=0)
|
||
print(subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
||
cwd=cwd, start_new_session=True, close_fds=True).pid)
|
||
PY
|
||
)"
|
||
printf '%s\n' "$pid" > "$RUN_DIR/$NAME.pid"
|
||
printf '%s\n' "$pid"
|
||
}
|
||
|
||
wait_http() { local url="$1" limit="${2:-120}" i=0
|
||
while [ "$i" -lt "$limit" ]; do
|
||
curl -fsS --max-time 2 -o /dev/null "$url" 2>/dev/null && return 0
|
||
sleep 1; i=$((i + 1))
|
||
done; return 1; }
|
||
|
||
# ------------------------------------------------------------------ install
|
||
install_backend_deps() {
|
||
head2 "后端依赖"
|
||
command -v python3 >/dev/null || die "缺少 python3(需要 3.11+)"
|
||
if [ ! -x "$PYBIN" ]; then
|
||
info "创建虚拟环境并安装依赖(首次 1~2 分钟)…"
|
||
python3 -m venv "$ROOT/.venv" || die "venv 创建失败(Debian/Ubuntu 需 apt install python3-venv)"
|
||
"$PYBIN" -m pip install -q --upgrade pip
|
||
fi
|
||
# 3.11+ 是硬门槛:Debian 12 自带的 python3 是 3.11(够用),但更老发行版的 3.9
|
||
# 会在 import 阶段就炸在 `str | None` 上。与其让服务起不来,不如装依赖时先说清楚。
|
||
"$PYBIN" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)' \
|
||
|| die "Python 版本过低($("$PYBIN" -V 2>&1)),本项目需要 3.11+,请装更新的 python3"
|
||
"$PYBIN" -m pip install -q -r "$ROOT/backend/requirements.txt" || die "后端依赖安装失败"
|
||
ok "$("$PYBIN" -V 2>&1) · fastapi/uvicorn/sqlalchemy 就绪"
|
||
if command -v ffmpeg >/dev/null && command -v ffprobe >/dev/null; then
|
||
ok "ffmpeg 就绪(合成、质检可用)"
|
||
else
|
||
warn "未找到 ffmpeg/ffprobe:合成与质检会失败(apt install ffmpeg / brew install ffmpeg)"
|
||
fi
|
||
}
|
||
|
||
build_frontend() {
|
||
head2 "前端构建"
|
||
command -v npm >/dev/null || die "缺少 npm(Node 18+)"
|
||
(cd "$ROOT/frontend" && { [ -d node_modules ] || npm ci --no-audit --no-fund; } && npm run build) \
|
||
> "$LOG_DIR/build.log" 2>&1 || { tail -30 "$LOG_DIR/build.log"; die "前端构建失败,详见 .local/logs/build.log"; }
|
||
[ -f "$ROOT/frontend/dist/index.html" ] || die "构建结束但没有 frontend/dist/index.html"
|
||
ok "frontend/dist 就绪($(du -sh "$ROOT/frontend/dist" | cut -f1),$(find "$ROOT/frontend/dist" -type f | wc -l | tr -d ' ') 个文件)"
|
||
}
|
||
|
||
fetch_asr_model() {
|
||
head2 "台词听写模型(whisper.cpp)"
|
||
if [ ! -x "$(command -v whisper-cli || true)" ] && [ ! -x "$(command -v whisper || true)" ]; then
|
||
warn "未安装 whisper-cli:台词听写质检(prompt_leak)不生效,其余功能不受影响"
|
||
info " macOS: brew install whisper-cpp Debian/Ubuntu: 从源码构建 https://github.com/ggml-org/whisper.cpp"
|
||
return 0
|
||
fi
|
||
if [ -f "$ASR_MODEL" ]; then ok "已存在 $(basename "$ASR_MODEL")($(du -sh "$ASR_MODEL" | cut -f1))"; return 0; fi
|
||
[ "$WANT_ASR" = 1 ] || { info "按 --no-asr 跳过(台词听写质检需要它)"; return 0; }
|
||
info "下载 ggml-small.bin(约 465MB,只下一次)…"
|
||
mkdir -p "$(dirname "$ASR_MODEL")"
|
||
curl -fL --retry 3 -o "$ASR_MODEL.tmp" "$ASR_URL" || die "下载失败:$ASR_URL"
|
||
mv "$ASR_MODEL.tmp" "$ASR_MODEL"
|
||
ok "$ASR_MODEL"
|
||
}
|
||
|
||
seed_data() {
|
||
head2 "表结构与初始数据"
|
||
if [ ! -d "$ROOT/scripts/seed" ]; then
|
||
# 从 git 克隆(而不是从部署包解压)时没有这个目录:不是错误,
|
||
# 应用首启会自己建表、播 mock、建 admin/admin1234,只是要在模型中心重配一遍。
|
||
warn "没有 scripts/seed/:跳过数据导入,服务器将用 mock 模型 + admin/admin1234 起步"
|
||
return 0
|
||
fi
|
||
"$PYBIN" "$ROOT/scripts/seed.py" install || die "导入模型配置与管理员失败"
|
||
info "${C_D}数据边界:表结构 + 模型配置 + 管理员。项目/镜头/媒体不在部署包里,服务器从空库开始。${C_0}"
|
||
}
|
||
|
||
# 单机 .env:不能直接 cp .env.example —— 那份模板是给 docker-compose 用的
|
||
# (Postgres + Redis + TASK_EXECUTOR=celery),照抄到一台裸机上会因为连不上 db
|
||
# 而起不来;server.sh 承诺的是「一台机器一条命令跑起来」,所以写 SQLite + inline。
|
||
ensure_env() {
|
||
[ -f "$ROOT/.env" ] && { ok ".env 就绪"; return 0; }
|
||
cat > "$ROOT/.env" <<'ENV'
|
||
# Nex Animation Studio · 单机部署配置(由 scripts/server.sh install 生成)
|
||
# 改完重启生效:./scripts/server.sh restart
|
||
|
||
# ---- 存储:单机用 SQLite,媒体落 ./media ----
|
||
DATABASE_URL=sqlite:///./nex_animation.db
|
||
MEDIA_ROOT=./media
|
||
|
||
# ---- 执行后端:inline = 进程内线程池,不需要 Redis ----
|
||
# 想横向扩展(多 GPU、多机)再切 celery,并另起 worker/beat,见 docker-compose.yml。
|
||
TASK_EXECUTOR=inline
|
||
|
||
# ---- 对外 ----
|
||
# 由后端一并托管前端产物:浏览器只访问一个端口。
|
||
SERVE_FRONTEND=on
|
||
AUTH_ENABLED=true
|
||
LOG_LEVEL=INFO
|
||
|
||
# ---- 模型 ----
|
||
# 真实模型端点写在模型中心(数据库)里;这里的值只在「首次播种」时生效,
|
||
# 已注册过的模型改这里不会覆盖库里的配置(见 app/providers/registry.py:seed_specs)。
|
||
COMFYUI_BASE_URL=
|
||
VIDEO_PROVIDER=comfyui
|
||
VIDEO_MODEL=minimax-h3
|
||
ASR_MODEL=models/asr/ggml-small.bin
|
||
ENV
|
||
warn "已生成单机版 .env(SQLite + inline);真实模型请填 COMFYUI_BASE_URL 或在模型中心直接配置"
|
||
}
|
||
|
||
do_install() {
|
||
printf 'Nex Animation Studio · 服务器部署安装(cwd=%s)\n' "$ROOT"
|
||
head2 "配置"
|
||
ensure_env
|
||
[ "$SKIP_DEPS" = 1 ] || install_backend_deps
|
||
[ "$SKIP_BUILD" = 1 ] || build_frontend
|
||
fetch_asr_model
|
||
seed_data
|
||
head2 "下一步"
|
||
info "1) 编辑 .env:COMFYUI_BASE_URL、DATABASE_URL(默认 SQLite,够用)、AUTH_ENABLED"
|
||
info " (模型配置已在导入时随包带过来:$(ls "$ROOT/scripts/seed" 2>/dev/null | tr '\n' ' '))"
|
||
info "2) ./scripts/server.sh start"
|
||
info "3) 浏览器打开 $BASE,用部署包里的管理员登录,随后立刻改密码"
|
||
info "4) ./scripts/server.sh verify # 用 mock/真实模型跑一遍全流程自检"
|
||
}
|
||
|
||
# ------------------------------------------------------------------ 运行
|
||
start_server() {
|
||
head2 "启动 :$PORT"
|
||
local pid busy
|
||
pid="$(pid_of)"
|
||
if alive "$pid"; then ok "已在运行 pid=$pid(跳过启动)"; return 0; fi
|
||
busy="$(port_pids "$PORT")"
|
||
[ -z "$busy" ] || die "端口 $PORT 被进程 $busy 占用;先 ./scripts/server.sh stop 或 --port 换端口"
|
||
[ -x "$PYBIN" ] || die "缺少 .venv:先 ./scripts/server.sh install"
|
||
[ -f "$ROOT/frontend/dist/index.html" ] || warn "没有 frontend/dist:本次只提供 API,页面请跑 ./scripts/server.sh build"
|
||
|
||
# SERVE_FRONTEND=on:由后端托管构建产物(见 backend/app/main.py 的 SPA 挂载)。
|
||
# --app-dir 从仓库根启动,.env / media / SQLite 相对路径全部锚定仓库根。
|
||
local cmd=(env SERVE_FRONTEND=on "$PYBIN" -m uvicorn app.main:app --app-dir "$ROOT/backend"
|
||
--host "$HOST" --port "$PORT")
|
||
pid="$(spawn "${cmd[@]}")"
|
||
if wait_http "$BASE/health" 120; then
|
||
ok "健康检查通过 pid=$pid"
|
||
else
|
||
tail -30 "$LOG_DIR/$NAME.log" 2>/dev/null || true
|
||
die "启动失败(120s 内 /health 未就绪)"
|
||
fi
|
||
if curl -fsS --max-time 3 -o /dev/null "$BASE/" 2>/dev/null; then
|
||
ok "页面 $BASE/"
|
||
else
|
||
warn "$BASE/ 未返回页面:确认已构建前端且 .env 没把 SERVE_FRONTEND 关掉"
|
||
fi
|
||
head2 "已启动"
|
||
info "${C_B}平台${C_0} $BASE/"
|
||
info "${C_B}接口${C_0} $BASE/docs"
|
||
local lan
|
||
lan="$( { iface="$(route -n get default 2>/dev/null | awk '/^[[:space:]]*interface:/{print $2; exit}')"
|
||
[ -n "$iface" ] && ipconfig getifaddr "$iface" 2>/dev/null; } || true )"
|
||
[ -n "$lan" ] && info "${C_B}局域网${C_0} http://$lan:$PORT/"
|
||
info "${C_B}日志${C_0} .local/logs/$NAME.log ${C_D}跟踪:./scripts/server.sh logs${C_0}"
|
||
info "${C_B}停止${C_0} ./scripts/server.sh stop"
|
||
}
|
||
|
||
stop_server() {
|
||
local pid; pid="$(pid_of)"
|
||
alive "$pid" || { rm -f "$RUN_DIR/$NAME.pid"; info "未在运行"; return 0; }
|
||
# pid 就是会话组长:按进程组回收,uvicorn 派生的 ffmpeg / whisper 子进程才不会被留下
|
||
kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
|
||
local i=0
|
||
while alive "$pid" && [ "$i" -lt 30 ]; do sleep 0.5; i=$((i + 1)); done
|
||
alive "$pid" && { kill -KILL -- "-$pid" 2>/dev/null || true; warn "已强制结束 pid=$pid"; }
|
||
rm -f "$RUN_DIR/$NAME.pid"
|
||
ok "已停止(pid=${pid})"
|
||
}
|
||
|
||
status_server() {
|
||
head2 "运行状态"
|
||
local pid; pid="$(pid_of)"
|
||
if alive "$pid"; then
|
||
printf ' %srunning%s server :%-6s pid=%s\n' "$C_G" "$C_0" "$PORT" "$pid"
|
||
elif [ -n "$(port_pids "$PORT")" ]; then
|
||
printf ' %sbusy%s server :%-6s 被本脚本之外的进程占用 (pid=%s)\n' \
|
||
"$C_Y" "$C_0" "$PORT" "$(port_pids "$PORT")"
|
||
else
|
||
printf ' %sstopped%s server :%s\n' "$C_D" "$C_0" "$PORT"
|
||
fi
|
||
[ -f "$ROOT/frontend/dist/index.html" ] && ok "前端产物 frontend/dist" || warn "前端产物缺失:./scripts/server.sh build"
|
||
[ -f "$ASR_MODEL" ] && ok "听写模型 $(basename "$ASR_MODEL")" || warn "听写模型缺失:台词听写质检不生效"
|
||
local comfy; comfy="$(env_value COMFYUI_BASE_URL)"
|
||
if [ -n "$comfy" ]; then
|
||
if curl -fsS --max-time 4 -o /dev/null "$comfy/system_stats" 2>/dev/null; then
|
||
printf ' %sreachable%s ComfyUI %s\n' "$C_G" "$C_0" "$comfy"
|
||
else
|
||
printf ' %sunreachable%s ComfyUI %s %s真实出片会失败%s\n' "$C_Y" "$C_0" "$comfy" "$C_D" "$C_0"
|
||
fi
|
||
fi
|
||
if curl -fsS --max-time 3 -o /dev/null "$BASE/health" 2>/dev/null; then
|
||
NEX_BASE="$BASE" "$PYBIN" <<'PY' || true
|
||
import os, sys, httpx
|
||
base = os.environ["NEX_BASE"].rstrip("/")
|
||
# 部署形态默认 AUTH_ENABLED=true,/api/* 要会话;未登录时这里会退化成
|
||
# 「HTTP 401 读不到模型」,看不出来是配置没导入还是没登录,所以先登录再读。
|
||
c = httpx.Client(base_url=base, timeout=10, trust_env=False)
|
||
h = c.get("/health").json()
|
||
print(f" executor={h.get('executor')} database={h.get('database')} ffmpeg={h.get('ffmpeg')} "
|
||
f"model_specs={h.get('model_specs')} frontend={h.get('frontend')} auth={h.get('auth_enabled')}")
|
||
try:
|
||
st = c.get("/api/auth/state").json()
|
||
if st.get("auth_enabled") and not st.get("authenticated"):
|
||
r = c.post("/api/auth/login", json={
|
||
"username": os.environ.get("NEX_USER") or (st.get("login_hint") or {}).get("username") or "admin",
|
||
"password": os.environ.get("NEX_PASS") or "admin1234"})
|
||
if r.status_code != 200:
|
||
# 登录不进去多半是首登强制改密没走界面,这里只提示,不阻塞 status
|
||
print(f" 模型注册表跳过:登录失败 {r.status_code}(改 NEX_USER/NEX_PASS 后重试)")
|
||
sys.exit(0)
|
||
for p in c.get("/api/providers").json()["providers"]:
|
||
if p.get("is_default"):
|
||
print(f" [默认] {p['media_type']:6s} {p['provider']}/{p['model']} 可用={p.get('available')}")
|
||
except Exception as exc:
|
||
print(f" 模型注册表读取失败:{type(exc).__name__}: {exc}")
|
||
PY
|
||
else
|
||
info "${C_D}服务未启动,健康检查与默认模型跳过${C_0}"
|
||
fi
|
||
}
|
||
|
||
verify_server() {
|
||
head2 "全流程自检(真实 HTTP + ffprobe 实测)"
|
||
wait_http "$BASE/health" 5 || die "$BASE 没有可用的后端"
|
||
NEX_BASE="$BASE" "$PYBIN" "$ROOT/scripts/e2e_selfcheck.py"
|
||
}
|
||
|
||
do_systemd() {
|
||
local unit user
|
||
user="${SUDO_USER:-$(id -un)}"
|
||
cat <<UNIT
|
||
# /etc/systemd/system/nex-animation.service
|
||
# 安装:sudo ./scripts/server.sh systemd > /etc/systemd/system/nex-animation.service
|
||
# sudo systemctl daemon-reload && sudo systemctl enable --now nex-animation
|
||
[Unit]
|
||
Description=Nex Animation Studio
|
||
After=network-online.target
|
||
Wants=network-online.target
|
||
|
||
[Service]
|
||
Type=simple
|
||
User=$user
|
||
WorkingDirectory=$ROOT
|
||
EnvironmentFile=$ROOT/.env
|
||
# SERVE_FRONTEND 之类已在 .env 里;这里显式再写一次,防止有人把 .env 换成 compose 版
|
||
# 前端产物由后端托管,对外一个端口;inline 执行器把调度守护线程跑在进程内,
|
||
# 所以只能一个进程——加 --workers 会变成 N 份守护线程重复派发。
|
||
ExecStart=$ROOT/.venv/bin/python -m uvicorn app.main:app --app-dir $ROOT/backend --host 0.0.0.0 --port $PORT
|
||
Restart=always
|
||
RestartSec=3
|
||
|
||
[Install]
|
||
WantedBy=multi-user.target
|
||
UNIT
|
||
}
|
||
|
||
usage() { sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; }
|
||
|
||
ACTION=start
|
||
while [ $# -gt 0 ]; do
|
||
case "$1" in
|
||
install|build|start|stop|restart|status|verify|logs|systemd) ACTION="$1" ;;
|
||
--port) PORT="${2:?--port 需要端口}"; shift ;;
|
||
--host) HOST="${2:?--host 需要地址}"; shift ;;
|
||
--db) DATABASE_URL="${2:?--db 需要连接串}"; export DATABASE_URL; shift ;;
|
||
--no-asr) WANT_ASR=0 ;;
|
||
--skip-deps) SKIP_DEPS=1 ;;
|
||
--skip-build) SKIP_BUILD=1 ;;
|
||
-h|--help) usage; exit 0 ;;
|
||
*) die "未知参数:$1(./scripts/server.sh --help)" ;;
|
||
esac
|
||
shift
|
||
done
|
||
|
||
# BASE 必须在解析完参数之后再算:早先算在文件头,`server.sh start --port 8020`
|
||
# 会把服务起在 8020、却去健康检查 8000,而那台机器 8000 上另有一个后端,
|
||
# 于是「健康检查通过」是个假绿灯。
|
||
case "$HOST" in 0.0.0.0|""|::|\[::\]) PROBE_HOST=127.0.0.1 ;; *) PROBE_HOST="$HOST" ;; esac
|
||
BASE="http://$PROBE_HOST:$PORT"
|
||
|
||
# 相对路径(models/asr、media、SQLite 文件名)一律锚到仓库根:
|
||
# 从 /etc/systemd 或 ~ 里启动服务时,cwd 不是项目目录,相对路径会指到别处。
|
||
cd "$ROOT"
|
||
if [ -n "$EXPLICIT_DB" ]; then export DATABASE_URL="$EXPLICIT_DB"; fi
|
||
|
||
case "$ACTION" in
|
||
install) do_install ;;
|
||
build) install_backend_deps; build_frontend ;;
|
||
start) start_server ;;
|
||
stop) stop_server ;;
|
||
restart) stop_server; start_server ;;
|
||
status) status_server ;;
|
||
verify) verify_server ;;
|
||
systemd) do_systemd ;;
|
||
logs) f="$LOG_DIR/$NAME.log"; [ -f "$f" ] || : > "$f"; exec tail -n 80 -F "$f" ;;
|
||
esac
|