Merge pull request '增加定时服务的健康检测' (#18) from damer into main

Reviewed-on: #18
2026-02-09 07:40:02 +00:00
parent 66241e5231 f03d36b5e9
commit 4578d503e7
7 changed files with 230 additions and 11 deletions
--- a/.gitea/workflows/server-deploy.yml
+++ b/.gitea/workflows/server-deploy.yml
@@ -56,6 +56,9 @@ jobs:
      # 健康检查路径（未配置则默认 /health；如果你没有 health 接口，可改为 /docs 或 /）
      HEALTHCHECK_PATH: ${{ vars.HEALTHCHECK_PATH }}

+      # 定时服务健康检查路径（检查 Redis/Worker/Beat；未配置则默认 /v1/push/scheduler/health）
+      SCHEDULER_HEALTHCHECK_PATH: ${{ vars.SCHEDULER_HEALTHCHECK_PATH }}
+
      # 可选：远端 env 文件路径（例如 /opt/mindfulness-server/.env.prod），存在则 docker run --env-file
      REMOTE_ENV_FILE: ${{ vars.REMOTE_ENV_FILE }}

@@ -258,11 +261,12 @@ jobs:
          GREEN_PORT="${GREEN_PORT:-8002}"
          CONTAINER_PORT="${CONTAINER_PORT:-8000}"
          HEALTHCHECK_PATH="${HEALTHCHECK_PATH:-/health}"
+          SCHEDULER_HEALTHCHECK_PATH="${SCHEDULER_HEALTHCHECK_PATH:-/v1/push/scheduler/health}"

          echo "准备部署：${DOCKER_IMAGE}:${DEPLOY_TAG} -> ${DEPLOY_ENV} (${SSH_USER}@${SSH_HOST}:${SSH_PORT})"

          ssh -p "${SSH_PORT}" -i ~/.ssh/id_rsa -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "${SSH_USER}@${SSH_HOST}" bash -s -- \
-            "${DOCKER_IMAGE}" "${DEPLOY_TAG}" "${NGINX_UPSTREAM_FILE}" "${NGINX_UPSTREAM_NAME}" "${BLUE_PORT}" "${GREEN_PORT}" "${CONTAINER_PORT}" "${HEALTHCHECK_PATH}" "${REMOTE_ENV_FILE:-}" <<'REMOTE'
+            "${DOCKER_IMAGE}" "${DEPLOY_TAG}" "${NGINX_UPSTREAM_FILE}" "${NGINX_UPSTREAM_NAME}" "${BLUE_PORT}" "${GREEN_PORT}" "${CONTAINER_PORT}" "${HEALTHCHECK_PATH}" "${SCHEDULER_HEALTHCHECK_PATH}" "${REMOTE_ENV_FILE:-}" <<'REMOTE'
          set -euo pipefail

          IMAGE="$1"
@@ -273,7 +277,8 @@ jobs:
          GREEN_PORT="$6"
          CONTAINER_PORT="$7"
          HEALTHCHECK_PATH="$8"
-          REMOTE_ENV_FILE="$9"
+          SCHEDULER_HEALTHCHECK_PATH="$9"
+          REMOTE_ENV_FILE="${10}"

          APP_DIR="/opt/mindfulness-server"
          ACTIVE_FILE="${APP_DIR}/active_color"
@@ -307,9 +312,18 @@ jobs:
          # 拉取镜像
          ${SUDO} docker pull "${IMAGE}:${TAG}"

-          # 启动新颜色容器
+          # 启动新颜色容器（API + Worker + Beat）
+          API_NAME="mindfulness-server-api-${NEW_COLOR}"
+          WORKER_NAME="mindfulness-server-worker-${NEW_COLOR}"
+          BEAT_NAME="mindfulness-server-beat-${NEW_COLOR}"
+
+          # 兼容旧命名：之前可能只有一个 mindfulness-server-blue/green
          ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" >/dev/null 2>&1 || true

+          ${SUDO} docker rm -f "${API_NAME}" >/dev/null 2>&1 || true
+          ${SUDO} docker rm -f "${WORKER_NAME}" >/dev/null 2>&1 || true
+          ${SUDO} docker rm -f "${BEAT_NAME}" >/dev/null 2>&1 || true
+
          ENV_FILE_ARGS=()
          if [[ -n "${REMOTE_ENV_FILE}" && -f "${REMOTE_ENV_FILE}" ]]; then
            ENV_FILE_ARGS=(--env-file "${REMOTE_ENV_FILE}")
@@ -318,11 +332,29 @@ jobs:
            echo "提示：REMOTE_ENV_FILE 已配置但文件不存在：${REMOTE_ENV_FILE}（将忽略 env-file）"
          fi

+          # API（对外暴露端口，仅该容器参与蓝绿切流）
          ${SUDO} docker run -d \
-            --name "mindfulness-server-${NEW_COLOR}" \
+            --name "${API_NAME}" \
            --restart=always \
            -p "${NEW_PORT}:${CONTAINER_PORT}" \
            "${ENV_FILE_ARGS[@]}" \
+            -e START_API=1 -e START_WORKER=0 -e START_BEAT=0 \
+            "${IMAGE}:${TAG}"
+
+          # Worker（处理异步/ETA 任务，不暴露端口）
+          ${SUDO} docker run -d \
+            --name "${WORKER_NAME}" \
+            --restart=always \
+            "${ENV_FILE_ARGS[@]}" \
+            -e START_API=0 -e START_WORKER=1 -e START_BEAT=0 \
+            "${IMAGE}:${TAG}"
+
+          # Beat（定时调度，不暴露端口）
+          ${SUDO} docker run -d \
+            --name "${BEAT_NAME}" \
+            --restart=always \
+            "${ENV_FILE_ARGS[@]}" \
+            -e START_API=0 -e START_WORKER=0 -e START_BEAT=1 \
            "${IMAGE}:${TAG}"

          # 健康检查
@@ -350,8 +382,37 @@ jobs:

            if [[ "$i" -eq 30 ]]; then
              echo "健康检查失败：新版本未就绪，回滚并退出"
-              ${SUDO} docker logs --tail 200 "mindfulness-server-${NEW_COLOR}" || true
-              ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+              ${SUDO} docker logs --tail 200 "${API_NAME}" || true
+              ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
+              exit 1
+            fi
+            sleep 2
+          done
+
+          # 定时服务健康检查（确保 Redis/Worker/Beat 都 OK，避免“接口正常但定时任务没跑”）
+          if [[ "${SCHEDULER_HEALTHCHECK_PATH}" != /* ]]; then
+            SCHEDULER_HEALTHCHECK_PATH="/${SCHEDULER_HEALTHCHECK_PATH}"
+          fi
+          SCHED_URL="http://127.0.0.1:${NEW_PORT}${SCHEDULER_HEALTHCHECK_PATH}"
+          echo "定时服务健康检查：${SCHED_URL}"
+
+          # Beat 心跳是按分钟刷新，这里最多等 90 秒（45*2s）
+          for i in $(seq 1 45); do
+            RES="$(curl -fsS "${SCHED_URL}" 2>/dev/null || true)"
+            if [[ -n "${RES}" ]] \
+              && echo "${RES}" | grep -q '"redis":{"ok":true' \
+              && echo "${RES}" | grep -q '"worker":{"ok":true' \
+              && echo "${RES}" | grep -q '"beat":{"ok":true' ; then
+              echo "定时服务健康检查通过：${RES}"
+              break
+            fi
+
+            if [[ "$i" -eq 45 ]]; then
+              echo "定时服务健康检查失败：${RES}"
+              echo "Worker/Beat 日志（各 120 行）："
+              ${SUDO} docker logs --tail 120 "${WORKER_NAME}" || true
+              ${SUDO} docker logs --tail 120 "${BEAT_NAME}" || true
+              ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
              exit 1
            fi
            sleep 2
@@ -360,24 +421,24 @@ jobs:
          # 切换 Nginx upstream（在同一个 conf 文件中通过 backup 做主备切换）
          if [[ ! -f "${UPSTREAM_FILE}" ]]; then
            echo "未找到 Nginx upstream 配置文件：${UPSTREAM_FILE}"
-            ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+            ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
            exit 1
          fi

          if ! ${SUDO} grep -qE "upstream[[:space:]]+${UPSTREAM_NAME}[[:space:]]*\\{" "${UPSTREAM_FILE}"; then
            echo "在 ${UPSTREAM_FILE} 中未找到 upstream：${UPSTREAM_NAME}"
-            ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+            ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
            exit 1
          fi

          if ! ${SUDO} grep -qE "server[[:space:]]+127\\.0\\.0\\.1:${BLUE_PORT}" "${UPSTREAM_FILE}"; then
            echo "在 ${UPSTREAM_FILE} 中未找到 server 127.0.0.1:${BLUE_PORT}（请先按参考配置写入 upstream）"
-            ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+            ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
            exit 1
          fi
          if ! ${SUDO} grep -qE "server[[:space:]]+127\\.0\\.0\\.1:${GREEN_PORT}" "${UPSTREAM_FILE}"; then
            echo "在 ${UPSTREAM_FILE} 中未找到 server 127.0.0.1:${GREEN_PORT}（请先按参考配置写入 upstream）"
-            ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+            ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
            exit 1
          fi

@@ -416,7 +477,7 @@ jobs:
            echo "Nginx 配置校验失败，回滚 upstream 配置并退出"
            ${SUDO} cp -f "${BACKUP_FILE}" "${UPSTREAM_FILE}" || true
            ${SUDO} nginx -t && ${SUDO} nginx -s reload || true
-            ${SUDO} docker rm -f "mindfulness-server-${NEW_COLOR}" || true
+            ${SUDO} docker rm -f "${API_NAME}" "${WORKER_NAME}" "${BEAT_NAME}" >/dev/null 2>&1 || true
            exit 1
          fi

@@ -424,6 +485,11 @@ jobs:
          echo "${NEW_COLOR}" | ${SUDO} tee "${ACTIVE_FILE}" >/dev/null

          # 下线旧容器（切流后再停旧的）
+          OLD_API_NAME="mindfulness-server-api-${OLD_COLOR}"
+          OLD_WORKER_NAME="mindfulness-server-worker-${OLD_COLOR}"
+          OLD_BEAT_NAME="mindfulness-server-beat-${OLD_COLOR}"
+          ${SUDO} docker rm -f "${OLD_API_NAME}" "${OLD_WORKER_NAME}" "${OLD_BEAT_NAME}" >/dev/null 2>&1 || true
+          # 兼容旧命名
          ${SUDO} docker rm -f "mindfulness-server-${OLD_COLOR}" >/dev/null 2>&1 || true

          echo "部署完成：${NEW_COLOR} 已上线"
--- a/client/eas.json
+++ b/client/eas.json
@@ -0,0 +1,21 @@
+{
+  "cli": {
+    "version": ">= 16.32.0",
+    "appVersionSource": "remote"
+  },
+  "build": {
+    "development": {
+      "developmentClient": true,
+      "distribution": "internal"
+    },
+    "preview": {
+      "distribution": "internal"
+    },
+    "production": {
+      "autoIncrement": true
+    }
+  },
+  "submit": {
+    "production": {}
+  }
+}
--- a/server/app/api/v1/push.py
+++ b/server/app/api/v1/push.py
@@ -4,6 +4,7 @@ from datetime import datetime, timezone
 from typing import Any, Literal, Optional

 import httpx
+import redis
 from fastapi import APIRouter, Depends, Header, HTTPException, Query
 from pydantic import BaseModel, Field
 from sqlalchemy import select
@@ -13,8 +14,10 @@ from app.api.limits import rate_limit_push_by_ip
 from app.core.config import get_settings
 from app.db.models.push_preference import PushPreference
 from app.db.models.push_token import PushToken
+from app.db.models.push_send_log import PushSendLog
 from app.db.session import get_db
 from app.features.user_profile_scoring.types import UserProfileV1_2
+from app.worker import celery_app


 router = APIRouter(
@@ -260,3 +263,84 @@ async def test_push(
    _ = accept_language
    return {"status": "ok", "expo": expo_res}

+
+def _env_prefix(app_env: str) -> str:
+    """
+    根据环境生成前缀：
+    - dev -> dev
+    - prod -> pro
+    """
+
+    return "dev" if str(app_env) == "dev" else "pro"
+
+
+@router.get("/scheduler/health")
+async def scheduler_health(db: AsyncSession = Depends(get_db)) -> dict[str, Any]:
+    """
+    推送“定时服务”健康检查（用于容器内验证）。
+
+    返回内容（尽量不暴露敏感信息）：
+    - Redis：是否可连通
+    - Worker：是否至少有一个 worker 在线（inspect ping）
+    - Beat：是否在跑（beat 心跳 key 是否在持续刷新）
+    - DB：是否可查询到 push_send_log 的最新时间（辅助定位排程是否生成）
+    """
+
+    settings = get_settings()
+    prefix = _env_prefix(settings.app_env)
+    beat_key = f"{prefix}:beat:heartbeat"
+
+    out: dict[str, Any] = {
+        "env": settings.app_env,
+        "redis": {"ok": False},
+        "worker": {"ok": False, "worker_count": 0},
+        "beat": {"ok": False, "last_heartbeat_at": None, "age_seconds": None},
+        "db": {"ok": False, "push_send_log_latest_created_at": None},
+        "now_utc": datetime.now(timezone.utc).isoformat(),
+    }
+
+    # 1) Redis 连通性 + 读取 beat 心跳
+    try:
+        r = redis.Redis.from_url(settings.celery_broker_url, decode_responses=True)
+        r.ping()
+        out["redis"]["ok"] = True
+
+        hb = r.get(beat_key)
+        if hb:
+            out["beat"]["last_heartbeat_at"] = hb
+            try:
+                # Python 3.11+ 支持解析 ISO8601（含 +00:00）
+                hb_dt = datetime.fromisoformat(hb.replace("Z", "+00:00"))
+                now = datetime.now(timezone.utc)
+                age = int((now - hb_dt.astimezone(timezone.utc)).total_seconds())
+                out["beat"]["age_seconds"] = age
+                # 2 分钟内认为健康（beat 每分钟刷新一次）
+                out["beat"]["ok"] = age <= 120
+            except Exception:
+                # 解析失败：至少说明 key 存在，但时间格式异常
+                out["beat"]["ok"] = False
+    except Exception as e:
+        out["redis"]["error"] = f"{type(e).__name__}: {e}"
+
+    # 2) Worker 在线性（inspect ping）
+    try:
+        insp = celery_app.control.inspect(timeout=1.0)
+        pings = insp.ping() or {}
+        if isinstance(pings, dict):
+            out["worker"]["worker_count"] = len(pings)
+            out["worker"]["ok"] = len(pings) > 0
+    except Exception as e:
+        out["worker"]["error"] = f"{type(e).__name__}: {e}"
+
+    # 3) DB：查询 push_send_log 最新创建时间（用于判断排程是否有生成）
+    try:
+        q = select(PushSendLog.created_at).order_by(PushSendLog.created_at.desc()).limit(1)
+        row = await db.execute(q)
+        latest = row.scalar_one_or_none()
+        out["db"]["ok"] = True
+        out["db"]["push_send_log_latest_created_at"] = latest.isoformat() if latest else None
+    except Exception as e:
+        out["db"]["error"] = f"{type(e).__name__}: {e}"
+
+    return out
+
--- a/server/app/tasks/init.py
+++ b/server/app/tasks/init.py
@@ -10,4 +10,5 @@ Celery 任务集合。
 from app.tasks import ping as _ping  # noqa: F401
 from app.tasks import reco as _reco  # noqa: F401
 from app.tasks import push as _push  # noqa: F401
+from app.tasks import ops as _ops  # noqa: F401

--- a/server/app/tasks/ops.py
+++ b/server/app/tasks/ops.py
@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import redis
+from celery import shared_task
+
+from app.core.config import get_settings
+
+
+def _env_prefix(app_env: str) -> str:
+    """
+    根据环境生成前缀：
+    - dev -> dev
+    - prod -> pro
+    """
+
+    return "dev" if str(app_env) == "dev" else "pro"
+
+
+@shared_task(name="tasks.ops.beat_heartbeat")
+def beat_heartbeat() -> dict[str, str]:
+    """
+    Beat 心跳任务（用于健康检查）。
+
+    作用：
+    - 由 Celery Beat 每分钟触发一次
+    - 写入 Redis 心跳 key，并设置 TTL
+    - API 侧读取该 key，可判断 beat 是否在运行
+    """
+
+    settings = get_settings()
+    prefix = _env_prefix(settings.app_env)
+    key = f"{prefix}:beat:heartbeat"
+    now = datetime.now(timezone.utc).isoformat()
+
+    r = redis.Redis.from_url(settings.celery_broker_url, decode_responses=True)
+    # TTL 设短一些：一旦 beat 挂了，很快就能从“过期/缺失”判断出来
+    r.set(key, now, ex=180)
+    return {"status": "ok", "key": key, "at": now}
+
--- a/server/app/worker.py
+++ b/server/app/worker.py
@@ -45,6 +45,12 @@ celery_app.conf.update(
 # - 这里按 UTC 00:10 触发一次；具体时间可按运维习惯调整
 celery_app.conf.timezone = "UTC"
 celery_app.conf.beat_schedule = {
+    # Beat 心跳：用于 API 健康检查判断 beat 是否在跑
+    "ops-beat-heartbeat": {
+        "task": "tasks.ops.beat_heartbeat",
+        "schedule": crontab(minute="*/1"),
+        "options": {"queue": f"{prefix}:celery"},
+    },
    "push-generate-daily-schedule": {
        "task": "tasks.push.generate_daily_schedule",
        "schedule": crontab(minute=10, hour=0),
--- a/server/celerybeat-schedule
+++ b/server/celerybeat-schedule