增加定时服务的健康检测
This commit is contained in:
吕新雨
@@ -56,6 +56,9 @@ jobs:
|
||||
# 健康检查路径（未配置则默认 /health；如果你没有 health 接口，可改为 /docs 或 /）
|
||||
HEALTHCHECK_PATH: ${{ vars.HEALTHCHECK_PATH }}
|
||||
|
||||
# 定时服务健康检查路径（检查 Redis/Worker/Beat；未配置则默认 /v1/push/scheduler/health）
|
||||
SCHEDULER_HEALTHCHECK_PATH: ${{ vars.SCHEDULER_HEALTHCHECK_PATH }}
|
||||
|
||||
# 可选：远端 env 文件路径（例如 /opt/mindfulness-server/.env.prod），存在则 docker run --env-file
|
||||
REMOTE_ENV_FILE: ${{ vars.REMOTE_ENV_FILE }}
|
||||
|
||||
@@ -258,11 +261,12 @@ jobs:
|
||||
GREEN_PORT="${GREEN_PORT:-8002}"
|
||||
CONTAINER_PORT="${CONTAINER_PORT:-8000}"
|
||||
HEALTHCHECK_PATH="${HEALTHCHECK_PATH:-/health}"
|
||||
SCHEDULER_HEALTHCHECK_PATH="${SCHEDULER_HEALTHCHECK_PATH:-/v1/push/scheduler/health}"
|
||||
|
||||
echo "准备部署：${DOCKER_IMAGE}:${DEPLOY_TAG} -> ${DEPLOY_ENV} (${SSH_USER}@${SSH_HOST}:${SSH_PORT})"
|
||||
|
||||
ssh -p "${SSH_PORT}" -i ~/.ssh/id_rsa -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "${SSH_USER}@${SSH_HOST}" bash -s -- \
|
||||
"${DOCKER_IMAGE}" "${DEPLOY_TAG}" "${NGINX_UPSTREAM_FILE}" "${NGINX_UPSTREAM_NAME}" "${BLUE_PORT}" "${GREEN_PORT}" "${CONTAINER_PORT}" "${HEALTHCHECK_PATH}" "${REMOTE_ENV_FILE:-}" <<'REMOTE'
|
||||
"${DOCKER_IMAGE}" "${DEPLOY_TAG}" "${NGINX_UPSTREAM_FILE}" "${NGINX_UPSTREAM_NAME}" "${BLUE_PORT}" "${GREEN_PORT}" "${CONTAINER_PORT}" "${HEALTHCHECK_PATH}" "${SCHEDULER_HEALTHCHECK_PATH}" "${REMOTE_ENV_FILE:-}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="$1"
|
||||
@@ -273,7 +277,8 @@ jobs:
|
||||
GREEN_PORT="$6"
|
||||
CONTAINER_PORT="$7"
|
||||
HEALTHCHECK_PATH="$8"
|
||||
REMOTE_ENV_FILE="$9"
|
||||
SCHEDULER_HEALTHCHECK_PATH="$9"
|
||||
REMOTE_ENV_FILE="${10}"
|
||||
|
||||
APP_DIR="/opt/mindfulness-server"
|
||||
ACTIVE_FILE="${APP_DIR}/active_color"
|
||||
@@ -307,9 +312,18 @@ jobs:
|
||||
# 拉取镜像
|
||||
${SUDO} docker pull "${IMAGE}:${TAG}"
|
||||
|
||||
# 启动新颜色容器
|
||||
# 启动新颜色容器（API + Worker + Beat）
|
||||
API_NAME="mindfulness-server-api-${NEW_COLOR}"
|
||||
WORKER_NAME="mindfulness-server-worker-${NEW_COLOR}"
|
||||
BEAT_NAME="mindfulness-server-beat-${NEW_COLOR}"
|
||||
|
||||
# 兼容旧命名：之前可能只有一个 mindfulness-server-blue/green
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" >/dev/null 2>&1 || true
|
||||
|
||||
${SUDO} docker rm -f "${API_NAME}" >/dev/null 2>&1 || true
|
||||
${SUDO} docker rm -f "${WORKER_NAME}" >/dev/null 2>&1 || true
|
||||
${SUDO} docker rm -f "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
|
||||
ENV_FILE_ARGS=()
|
||||
if [[ -n "${REMOTE_ENV_FILE}" && -f "${REMOTE_ENV_FILE}" ]]; then
|
||||
ENV_FILE_ARGS=(--env-file "${REMOTE_ENV_FILE}")
|
||||
@@ -318,11 +332,29 @@ jobs:
|
||||
echo "提示：REMOTE_ENV_FILE 已配置但文件不存在：${REMOTE_ENV_FILE}（将忽略 env-file）"
|
||||
fi
|
||||
|
||||
# API（对外暴露端口，仅该容器参与蓝绿切流）
|
||||
${SUDO} docker run -d \
|
||||
--name "mindfulness-server-${NEW_COLOR}" \
|
||||
--name "${API_NAME}" \
|
||||
--restart=always \
|
||||
-p "${NEW_PORT}:${CONTAINER_PORT}" \
|
||||
"${ENV_FILE_ARGS[@]}" \
|
||||
-e START_API=1 -e START_WORKER=0 -e START_BEAT=0 \
|
||||
"${IMAGE}:${TAG}"
|
||||
|
||||
# Worker（处理异步/ETA 任务，不暴露端口）
|
||||
${SUDO} docker run -d \
|
||||
--name "${WORKER_NAME}" \
|
||||
--restart=always \
|
||||
"${ENV_FILE_ARGS[@]}" \
|
||||
-e START_API=0 -e START_WORKER=1 -e START_BEAT=0 \
|
||||
"${IMAGE}:${TAG}"
|
||||
|
||||
# Beat（定时调度，不暴露端口）
|
||||
${SUDO} docker run -d \
|
||||
--name "${BEAT_NAME}" \
|
||||
--restart=always \
|
||||
"${ENV_FILE_ARGS[@]}" \
|
||||
-e START_API=0 -e START_WORKER=0 -e START_BEAT=1 \
|
||||
"${IMAGE}:${TAG}"
|
||||
|
||||
# 健康检查
|
||||
@@ -350,8 +382,37 @@ jobs:
|
||||
|
||||
if [[ "$i" -eq 30 ]]; then
|
||||
echo "健康检查失败：新版本未就绪，回滚并退出"
|
||||
${SUDO} docker logs --tail 200 "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker logs --tail 200 "${API_NAME}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# 定时服务健康检查（确保 Redis/Worker/Beat 都 OK，避免“接口正常但定时任务没跑”）
|
||||
if [[ "${SCHEDULER_HEALTHCHECK_PATH}" != /* ]]; then
|
||||
SCHEDULER_HEALTHCHECK_PATH="/${SCHEDULER_HEALTHCHECK_PATH}"
|
||||
fi
|
||||
SCHED_URL="http://127.0.0.1:${NEW_PORT}${SCHEDULER_HEALTHCHECK_PATH}"
|
||||
echo "定时服务健康检查：${SCHED_URL}"
|
||||
|
||||
# Beat 心跳是按分钟刷新，这里最多等 90 秒（45*2s）
|
||||
for i in $(seq 1 45); do
|
||||
RES="$(curl -fsS "${SCHED_URL}" 2>/dev/null || true)"
|
||||
if [[ -n "${RES}" ]] \
|
||||
&& echo "${RES}" | grep -q '"redis":{"ok":true' \
|
||||
&& echo "${RES}" | grep -q '"worker":{"ok":true' \
|
||||
&& echo "${RES}" | grep -q '"beat":{"ok":true' ; then
|
||||
echo "定时服务健康检查通过：${RES}"
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ "$i" -eq 45 ]]; then
|
||||
echo "定时服务健康检查失败：${RES}"
|
||||
echo "Worker/Beat 日志（各 120 行）："
|
||||
${SUDO} docker logs --tail 120 "${WORKER_NAME}" || true
|
||||
${SUDO} docker logs --tail 120 "${BEAT_NAME}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
@@ -360,24 +421,24 @@ jobs:
|
||||
# 切换 Nginx upstream（在同一个 conf 文件中通过 backup 做主备切换）
|
||||
if [[ ! -f "${UPSTREAM_FILE}" ]]; then
|
||||
echo "未找到 Nginx upstream 配置文件：${UPSTREAM_FILE}"
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ${SUDO} grep -qE "upstream[[:space:]]+${UPSTREAM_NAME}[[:space:]]*\\{" "${UPSTREAM_FILE}"; then
|
||||
echo "在 ${UPSTREAM_FILE} 中未找到 upstream：${UPSTREAM_NAME}"
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ${SUDO} grep -qE "server[[:space:]]+127\\.0\\.0\\.1:${BLUE_PORT}" "${UPSTREAM_FILE}"; then
|
||||
echo "在 ${UPSTREAM_FILE} 中未找到 server 127.0.0.1:${BLUE_PORT}（请先按参考配置写入 upstream）"
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
if ! ${SUDO} grep -qE "server[[:space:]]+127\\.0\\.0\\.1:${GREEN_PORT}" "${UPSTREAM_FILE}"; then
|
||||
echo "在 ${UPSTREAM_FILE} 中未找到 server 127.0.0.1:${GREEN_PORT}（请先按参考配置写入 upstream）"
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -416,7 +477,7 @@ jobs:
|
||||
echo "Nginx 配置校验失败，回滚 upstream 配置并退出"
|
||||
${SUDO} cp -f "${BACKUP_FILE}" "${UPSTREAM_FILE}" || true
|
||||
${SUDO} nginx -t && ${SUDO} nginx -s reload || true
|
||||
${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
|
||||
${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -424,6 +485,11 @@ jobs:
|
||||
echo "${NEW_COLOR}" | ${SUDO} tee "${ACTIVE_FILE}" >/dev/null
|
||||
|
||||
# 下线旧容器（切流后再停旧的）
|
||||
OLD_API_NAME="mindfulness-server-api-${OLD_COLOR}"
|
||||
OLD_WORKER_NAME="mindfulness-server-worker-${OLD_COLOR}"
|
||||
OLD_BEAT_NAME="mindfulness-server-beat-${OLD_COLOR}"
|
||||
${SUDO} docker rm -f "${OLD_API_NAME}" "${OLD_WORKER_NAME}" "${OLD_BEAT_NAME}" >/dev/null 2>&1 || true
|
||||
# 兼容旧命名
|
||||
${SUDO} docker rm -f "mindfulness-server-${OLD_COLOR}" >/dev/null 2>&1 || true
|
||||
|
||||
echo "部署完成：${NEW_COLOR} 已上线"
|
||||
|
||||
Reference in New Issue
Block a user