修复：定时任务

2026-02-11 11:14:51 +08:00
parent decc7f9564
commit 402cbf90eb
6 changed files with 172 additions and 94 deletions
--- a/client/ios/Podfile.lock
+++ b/client/ios/Podfile.lock
@@ -2540,30 +2540,30 @@ EXTERNAL SOURCES:
    :path: "../node_modules/react-native/ReactCommon/yoga"

 SPEC CHECKSUMS:
-  EXApplication: 1e98d4b1dccdf30627f92917f4b2c5a53c330e5f
-  EXConstants: fce59a631a06c4151602843667f7cfe35f81e271
+  EXApplication: 13420f8139864183f8a04fd6099077bdf8cfb186
+  EXConstants: 3feb66fd1d94202fc1f0946d74e029d8b224b60e
  EXJSONUtils: 1d3e4590438c3ee593684186007028a14b3686cd
-  EXManifests: a8d97683e5c7a3b026ffbd58559c64dc655b747b
-  EXNotifications: 9eec98712cc814ceff916d876cb53859003b0597
-  Expo: 4e503a041c59c4e34c8be262a135848ad5cd3710
-  expo-dev-client: 425ee077d6754a98cfe3a2e2410d29b440b24c9d
-  expo-dev-launcher: a4f4cdef064ab1fb8621e5b8c7c457cd6e9568c3
-  expo-dev-menu: 05b18812110c175814c6af0d09dd658abcc5e00d
+  EXManifests: 83ef0844fcf06d6099b12a7bdbd7d36fc0e1dd16
+  EXNotifications: 2a3feb7af6194828d9aafda72f63a9a03866230a
+  Expo: b8d64eb9a496ebe8c71e3dae7eeb7f394b146b80
+  expo-dev-client: 12ef7d5b14d93e309922acea78dcd851db583a87
+  expo-dev-launcher: 47994056008ffdc30a6a5e328a375b3e30a8db05
+  expo-dev-menu: ea4fb803ace52e60d7cd8060c7cd379612a140b2
  expo-dev-menu-interface: 600df12ea01efecdd822daaf13cc0ac091775533
-  ExpoAsset: f867e55ceb428aab99e1e8c082b5aee7c159ea18
-  ExpoCrypto: b6105ebaa15d6b38a811e71e43b52cd934945322
-  ExpoDevice: 6327c3c200816795708885adf540d26ecab83d1a
-  ExpoFileSystem: 858a44267a3e6e9057e0888ad7c7cfbf55d52063
-  ExpoFont: f543ce20a228dd702813668b1a07b46f51878d47
-  ExpoHead: 4425246bc93411f0fe7f6945f95f698e91db8780
-  ExpoKeepAwake: 55f75eca6499bb9e4231ebad6f3e9cb8f99c0296
-  ExpoLinearGradient: 809102bdb979f590083af49f7fa4805cd931bd58
-  ExpoLinking: 8f0aaf69aa56f832913030503b6263dc6f647f37
-  ExpoLocalization: d9168d5300a5b03e5e78b986124d11fb6ec3ebbd
-  ExpoModulesCore: f3da4f1ab5a8375d0beafab763739dbee8446583
-  ExpoSplashScreen: bc3cffefca2716e5f22350ca109badd7e50ec14d
-  ExpoWebBrowser: 17b064c621789e41d4816c95c93f429b84971f52
-  EXUpdatesInterface: 5adf50cb41e079c861da6d9b4b954c3db9a50734
+  ExpoAsset: d999f3bbd998a750f3b74cb913229848901b926b
+  ExpoCrypto: 4d23a9ff67c25e2ed23ca792d81e58817a7ea1b9
+  ExpoDevice: 0773c782b055558ca9b40b74aa4a8133a66cd0d2
+  ExpoFileSystem: aefcd337b94b874f88752ebefc52813b84992fad
+  ExpoFont: c625dbd97ed57e9089b172b2a7bb99003d074664
+  ExpoHead: b691a2ed7ab02ed820b6c6468941832d34969c29
+  ExpoKeepAwake: 44bf6715bc1d2ddb17afe19d927cd039cda123f0
+  ExpoLinearGradient: 814a21fc4056c3cf606e4f19e31e47074c5b5a86
+  ExpoLinking: ebf543fd411d56375cb4eee07f6ab4e31c7ad959
+  ExpoLocalization: 6ac6f326210f0a3141ef6f58ab8f8f4ed003b485
+  ExpoModulesCore: 77496909fd3c800f97f7f2007dd26aeac4bb3798
+  ExpoSplashScreen: 72fbc6dd9d6404dd9d0725a56c9ac1383bc0b14f
+  ExpoWebBrowser: 88b116cd378d9609c776c0903fe4070fca461588
+  EXUpdatesInterface: 1436757deb0d574b84bba063bd024c315e0ec08b
  FBLazyVector: e95a291ad2dadb88e42b06e0c5fb8262de53ec12
  hermes-engine: 9f4dfe93326146a1c99eb535b1cb0b857a3cd172
  RCTDeprecation: 943572d4be82d480a48f4884f670135ae30bf990
@@ -2571,72 +2571,72 @@ SPEC CHECKSUMS:
  RCTTypeSafety: 16a4144ca3f959583ab019b57d5633df10b5e97c
  React: 914f8695f9bf38e6418228c2ffb70021e559f92f
  React-callinvoker: 1c0808402aee0c6d4a0d8e7220ce6547af9fba71
-  React-Core: c61410ef0ca6055e204a963992e363227e0fd1c5
-  React-Core-prebuilt: 02f0ad625ddd47463c009c2d0c5dd35c0d982599
-  React-CoreModules: 1f6d1744b5f9f2ec684a4bb5ced25370f87e5382
-  React-cxxreact: 3af79478e8187b63ffc22b794cd42d3fc1f1f2da
+  React-Core: 4ae98f9e8135b8ddbd7c98730afb6fdae883db90
+  React-Core-prebuilt: 8f4cca589c14e8cf8fc6db4587ef1c2056b5c151
+  React-CoreModules: e878a90bb19b8f3851818af997dbae3b3b0a27ac
+  React-cxxreact: 28af9844f6dc87be1385ab521fbfb3746f19563c
  React-debug: 6328c2228e268846161f10082e80dc69eac2e90a
-  React-defaultsnativemodule: d635ef36d755321e5d6fc065bd166b2c5a0e9833
-  React-domnativemodule: dd28f6d96cd21236e020be2eff6fe0b7d4ec3b66
-  React-Fabric: 2e32c3fdbb1fbcf5fde54607e3abe453c6652ce2
-  React-FabricComponents: 5ed0cdb81f6b91656cb4d3be432feaa28a58071a
-  React-FabricImage: 2bc714f818cb24e454f5d3961864373271b2faf8
-  React-featureflags: 847642f41fa71ad4eec5e0351badebcad4fe6171
-  React-featureflagsnativemodule: c868a544b2c626fa337bcbd364b1befe749f0d3f
-  React-graphics: 192ec701def5b3f2a07db2814dfba5a44986cff6
-  React-hermes: e875778b496c86d07ab2ccaa36a9505d248a254b
-  React-idlecallbacksnativemodule: 4d57965cdf82c14ee3b337189836cd8491632b76
-  React-ImageManager: bd0b99e370b13de82c9cd15f0f08144ff3de079e
-  React-jserrorhandler: a2fdef4cbcfdcdf3fa9f5d1f7190f7fd4535248d
-  React-jsi: 89d43d1e7d4d0663f8ba67e0b39eb4e4672c27de
-  React-jsiexecutor: abe4874aaab90dfee5dec480680220b2f8af07e3
-  React-jsinspector: a0b3e051aef842b0b2be2353790ae2b2a5a65a8f
-  React-jsinspectorcdp: 6346013b2247c6263fbf5199adf4a8751e53bd89
-  React-jsinspectornetwork: 26281aa50d49fc1ec93abf981d934698fa95714f
-  React-jsinspectortracing: 55eedf6d57540507570259a778663b90060bbd6e
-  React-jsitooling: 0e001113fa56d8498aa8ac28437ac0d36348e51a
-  React-jsitracing: b713793eb8a5bbc4d86a84e9d9e5023c0f58cbaf
-  React-logger: 50fdb9a8236da90c0b1072da5c32ee03aeb5bf28
-  React-Mapbuffer: 9050ee10c19f4f7fca8963d0211b2854d624973e
-  React-microtasksnativemodule: f775db9e991c6f3b8ccbc02bfcde22770f96e23b
-  react-native-safe-area-context: 37e680fc4cace3c0030ee46e8987d24f5d3bdab2
-  React-NativeModulesApple: 8969913947d5b576de4ed371a939455a8daf28aa
+  React-defaultsnativemodule: afc9d809ec75780f39464a6949c07987fbea488c
+  React-domnativemodule: 91a233260411d41f27f67aa1358b7f9f0bfd101d
+  React-Fabric: 21f349b5e93f305a3c38c885902683a9c79cf983
+  React-FabricComponents: 47ac634cc9ecc64b30a9997192f510eebe4177e4
+  React-FabricImage: 21873acd6d4a51a0b97c133141051c7acb11cc86
+  React-featureflags: 653f469f0c3c9dc271d610373e3b6e66a9fd847d
+  React-featureflagsnativemodule: c91a8a3880e0f4838286402241ead47db43aed28
+  React-graphics: b4bdb0f635b8048c652a5d2b73eb8b1ddd950f24
+  React-hermes: fcfad3b917400f49026f3232561e039c9d1c34bf
+  React-idlecallbacksnativemodule: 8cb83207e39f8179ac1d344b6177c6ab3ccebcdc
+  React-ImageManager: 396128004783fc510e629124dce682d38d1088e7
+  React-jserrorhandler: b58b788d788cdbf8bda7db74a88ebfcffc8a0795
+  React-jsi: d2c3f8555175371c02da6dfe7ed1b64b55a9d6c0
+  React-jsiexecutor: ba537434eb45ee018b590ed7d29ee233fddb8669
+  React-jsinspector: f21b6654baf96cb9f71748844a32468a5f73ad51
+  React-jsinspectorcdp: 3f8be4830694c3c1c39442e50f8db877966d43f0
+  React-jsinspectornetwork: 70e41469565712ad60e11d9c8b8f999b9f7f61eb
+  React-jsinspectortracing: eccf9bfa4ec7f130d514f215cfb2222dc3c0e270
+  React-jsitooling: b376a695f5a507627f7934748533b24eed1751ca
+  React-jsitracing: 5c8c3273dda2d95191cc0612fb5e71c4d9018d2a
+  React-logger: c3e2f8a2e284341205f61eef3d4677ab5a309dfd
+  React-Mapbuffer: 603c18db65844bb81dbe62fee8fcc976eaeb7108
+  React-microtasksnativemodule: d77e0c426fce34c23227394c96ca1033b30c813c
+  react-native-safe-area-context: 53f796cb6c814661bbe99fbdfd0585d07b996cdd
+  React-NativeModulesApple: 1664340b8750d64e0ef3907c5e53d9481f74bcbd
  React-oscompat: ce47230ed20185e91de62d8c6d139ae61763d09c
-  React-perflogger: 02b010e665772c7dcb859d85d44c1bfc5ac7c0e4
-  React-performancetimeline: 130db956b5a83aa4fb41ddf5ae68da89f3fb1526
+  React-perflogger: b1af3cfb3f095f819b2814910000392a8e17ba9f
+  React-performancetimeline: f9ec65b77bcadbc7bd8b47a6f4b4b697da7b1490
  React-RCTActionSheet: 0b14875b3963e9124a5a29a45bd1b22df8803916
-  React-RCTAnimation: a7b90fd2af7bb9c084428867445a1481a8cb112e
-  React-RCTAppDelegate: 3262bedd01263f140ec62b7989f4355f57cec016
-  React-RCTBlob: c17531368702f1ebed5d0ada75a7cf5915072a53
-  React-RCTFabric: 6409edd8cfdc3133b6cc75636d3b858fdb1d11ea
-  React-RCTFBReactNativeSpec: c004b27b4fa3bd85878ad2cf53de3bbec85da797
-  React-RCTImage: c68078a120d0123f4f07a5ac77bea3bb10242f32
-  React-RCTLinking: cf8f9391fe7fe471f96da3a5f0435235eca18c5b
-  React-RCTNetwork: ca31f7c879355760c2d9832a06ee35f517938a20
-  React-RCTRuntime: a6cf4a1e42754fc87f493e538f2ac6b820e45418
-  React-RCTSettings: e0e140b2ff4bf86d34e9637f6316848fc00be035
-  React-RCTText: 75915bace6f7877c03a840cc7b6c622fb62bfa6b
-  React-RCTVibration: 25f26b85e5e432bb3c256f8b384f9269e9529f25
+  React-RCTAnimation: 60f6eca214a62b9673f64db6df3830cee902b5af
+  React-RCTAppDelegate: 37734b39bac108af30a0fd9d3e1149ec68b82c28
+  React-RCTBlob: 83fbcbd57755caf021787324aac2fe9b028cc264
+  React-RCTFabric: a05cb1df484008db3753c8b4a71e4c6d9f1e43a6
+  React-RCTFBReactNativeSpec: d58d7ae9447020bbbac651e3b0674422aba18266
+  React-RCTImage: 47aba3be7c6c64f956b7918ab933769602406aac
+  React-RCTLinking: 2dbaa4df2e4523f68baa07936bd8efdfa34d5f31
+  React-RCTNetwork: 1fca7455f9dedf7de2b95bec438da06680f3b000
+  React-RCTRuntime: 17819dd1dfc8613efaf4cbb9d8686baae4a83e5b
+  React-RCTSettings: 01bf91c856862354d3d2f642ccb82f3697a4284a
+  React-RCTText: cb576a3797dcb64933613c522296a07eaafc0461
+  React-RCTVibration: 560af8c086741f3525b8456a482cdbe27f9d098e
  React-rendererconsistency: 2dac03f448ff337235fd5820b10f81633328870d
-  React-renderercss: 477da167bb96b5ac86d30c5d295412fb853f5453
-  React-rendererdebug: 2a1798c6f3ef5f22d466df24c33653edbabb5b89
-  React-RuntimeApple: 28cf4d8eb18432f6a21abbed7d801ab7f6b6f0b4
-  React-RuntimeCore: 41bf0fd56a00de5660f222415af49879fa49c4f0
-  React-runtimeexecutor: 1afb774dde3011348e8334be69d2f57a359ea43e
-  React-RuntimeHermes: f3b158ea40e8212b1a723a68b4315e7a495c5fc6
-  React-runtimescheduler: 3e1e2bec7300bae512533107d8e54c6e5c63fe0f
-  React-timing: 6fa9883de2e41791e5dc4ec404e5e37f3f50e801
-  React-utils: 6e2035b53d087927768649a11a26c4e092448e34
-  ReactAppDependencyProvider: 1bcd3527ac0390a1c898c114f81ff954be35ed79
-  ReactCodegen: 7d4593f7591f002d137fe40cef3f6c11f13c88cc
-  ReactCommon: 08810150b1206cc44aecf5f6ae19af32f29151a8
+  React-renderercss: c5c6b7a15948dd28facca39a18ac269073718490
+  React-rendererdebug: 3c9d5e1634273f5a24d84cc5669f290ce0bdc812
+  React-RuntimeApple: 887637d1e12ea8262df7d32bc100467df2302613
+  React-RuntimeCore: 91f779835dc4f8f84777fe5dd24f1a22f96454e4
+  React-runtimeexecutor: 8bb6b738f37b0ada4a6269e6f8ab1133dea0285c
+  React-RuntimeHermes: 4cb93de9fa8b1cc753d200dbe61a01b9ec5f5562
+  React-runtimescheduler: 83dc28f530bfbd2fce84ed13aa7feebdc24e5af7
+  React-timing: 03c7217455d2bff459b27a3811be25796b600f47
+  React-utils: 6d46795ae0444ec8a5d9a5f201157b286bf5250a
+  ReactAppDependencyProvider: c277c5b231881ad4f00cd59e3aa0671b99d7ebee
+  ReactCodegen: 4c44b74b77fc41ae25b9e2c7e9bd6e2bc772c23f
+  ReactCommon: e6e232202a447d353e5531f2be82f50f47cbaa9a
  ReactNativeDependencies: 71ce9c28beb282aa720ea7b46980fff9669f428a
-  RNCAsyncStorage: 3a4f5e2777dae1688b781a487923a08569e27fe4
-  RNGestureHandler: e0d0bce5599f6120b7adf90c38d2805e2935795f
-  RNReanimated: e5c702a3e24cc1c68b2de67671713f35461678f4
-  RNScreens: d8d6f1792f6e7ac12b0190d33d8d390efc0c1845
-  RNSVG: 31d6639663c249b7d5abc9728dde2041eb2a3c34
-  RNWorklets: 76fce72926e28e304afb44f0da23b2d24f2c1fa0
+  RNCAsyncStorage: e85a99325df9eb0191a6ee2b2a842644c7eb29f4
+  RNGestureHandler: 40c2d1c168e54715fe52e0fb16cb38c54611e4f3
+  RNReanimated: 43f611f1c85c90e0273df7399bf1536f8e2bd125
+  RNScreens: dd61bc3a3e6f6901ad833efa411917d44827cf51
+  RNSVG: 2825ee146e0f6a16221e852299943e4cceef4528
+  RNWorklets: 28ee7370ca8da356fcc914e3e68b97e9752196d2
  Yoga: 5934998fbeaef7845dbf698f698518695ab4cd1a

 PODFILE CHECKSUM: c2c3838f0b2a579fef2350bff2ecaa005e27145d
--- a/server/app/tasks/push.py
+++ b/server/app/tasks/push.py
@@ -9,7 +9,7 @@ from zoneinfo import ZoneInfo

 import httpx
 from celery import current_app, shared_task
-from sqlalchemy import select
+from sqlalchemy import select, update

 from app.core.config import get_settings
 from app.db.models.push_preference import PushPreference
@@ -167,16 +167,25 @@ async def _generate_schedule_once(*, now_utc: datetime, max_users: int = 5000) -
                created += 1

                # 投递 ETA 发送任务
-                current_app.send_task(
-                    "tasks.push.send_scheduled",
-                    kwargs={
-                        "client_user_id": pref.client_user_id,
-                        "local_date": target.local_date.isoformat(),
-                        "slot_index": int(idx),
-                    },
-                    eta=dt_utc,
-                )
-                scheduled += 1
+                try:
+                    current_app.send_task(
+                        "tasks.push.send_scheduled",
+                        kwargs={
+                            "client_user_id": pref.client_user_id,
+                            "local_date": target.local_date.isoformat(),
+                            "slot_index": int(idx),
+                        },
+                        eta=dt_utc,
+                    )
+                    scheduled += 1
+                except Exception as e:
+                    # 关键：如果投递失败（例如 broker 短暂不可用），不要让 log 永远卡在 scheduled
+                    log.status = "failed"
+                    log.error = f"enqueue_failed:{type(e).__name__}"
+                    try:
+                        await session.commit()
+                    except Exception:
+                        await session.rollback()

    return {"created": created, "scheduled": scheduled}

@@ -209,6 +218,24 @@ async def _send_once_async(*, client_user_id: str, local_date: date, slot_index:
            return {"status": "noop", "reason": "no_log"}
        if str(log.status) == "sent":
            return {"status": "noop", "reason": "already_sent"}
+        if str(log.status) != "scheduled":
+            # 例如 failed/skipped/sending：不再重复尝试
+            return {"status": "noop", "reason": f"not_scheduled:{log.status}"}
+
+        # 原子抢占：避免同一条 scheduled 被重复发送（例如 ETA 任务丢失后被补偿重投递）
+        # 只有从 scheduled -> sending 抢占成功的任务才能继续执行。
+        res = await session.execute(
+            update(PushSendLog)
+            .where(
+                PushSendLog.id == log.id,
+                PushSendLog.status == "scheduled",
+            )
+            .values(status="sending", error=None)
+        )
+        await session.commit()
+        if (res.rowcount or 0) <= 0:
+            return {"status": "noop", "reason": "already_in_progress_or_processed"}
+        log.status = "sending"

        # 2) 当前偏好检查（用户可能中途关闭/改次数）
        qpref = select(PushPreference).where(PushPreference.client_user_id == client_user_id)
@@ -310,3 +337,46 @@ def send_scheduled(*, client_user_id: str, local_date: str, slot_index: int) ->
    d = date.fromisoformat(str(local_date))
    return asyncio.run(_send_once_async(client_user_id=str(client_user_id), local_date=d, slot_index=int(slot_index)))

+
+@shared_task(name="tasks.push.requeue_overdue")
+def requeue_overdue(*, grace_seconds: int = 300, limit: int = 200) -> dict[str, Any]:
+    """
+    补偿任务：扫描“已到时间但仍处于 scheduled”的记录并重新投递发送任务。
+
+    目的：
+    - 覆盖 broker 短暂不可用、worker 重启、ETA 任务丢失等导致的“scheduled 卡住”
+    - 与 send_scheduled 内部的原子状态抢占配合，避免重复发送
+    """
+
+    now_utc_naive = datetime.now(timezone.utc).replace(tzinfo=None)
+    cutoff = now_utc_naive - timedelta(seconds=int(grace_seconds))
+
+    async def _run() -> dict[str, Any]:
+        requeued = 0
+        async with AsyncSessionLocal() as session:
+            q = (
+                select(PushSendLog)
+                .where(PushSendLog.status == "scheduled", PushSendLog.scheduled_at <= cutoff)
+                .order_by(PushSendLog.scheduled_at.asc())
+                .limit(int(limit))
+            )
+            rows = await session.execute(q)
+            logs = list(rows.scalars().all())
+            for log in logs:
+                try:
+                    current_app.send_task(
+                        "tasks.push.send_scheduled",
+                        kwargs={
+                            "client_user_id": str(log.client_user_id),
+                            "local_date": str(log.local_date),
+                            "slot_index": int(log.slot_index),
+                        },
+                    )
+                    requeued += 1
+                except Exception:
+                    # 忽略单条投递失败，交给下一轮补偿
+                    continue
+        return {"status": "ok", "requeued": requeued, "cutoff": cutoff.isoformat()}
+
+    return asyncio.run(_run())
+
--- a/server/app/worker.py
+++ b/server/app/worker.py
@@ -57,6 +57,14 @@ celery_app.conf.beat_schedule = {
        "kwargs": {"max_users": 5000},
        "options": {"queue": f"{prefix}:celery"},
    }
+    ,
+    # 补偿：每 5 分钟扫描一次 overdue scheduled 并重投递
+    "push-requeue-overdue": {
+        "task": "tasks.push.requeue_overdue",
+        "schedule": crontab(minute="*/5"),
+        "kwargs": {"grace_seconds": 300, "limit": 200},
+        "options": {"queue": f"{prefix}:celery"},
+    },
 }

 # 自动发现任务（约定：导入 app.tasks 触发其内部对子模块的显式导入）
--- a/server/celerybeat-schedule
+++ b/server/celerybeat-schedule
--- a/server/celerybeat-schedule-shm
+++ b/server/celerybeat-schedule-shm
--- a/server/celerybeat-schedule-wal
+++ b/server/celerybeat-schedule-wal