新功能:个性化推荐算法

This commit is contained in:
吕新雨
2026-02-02 16:47:37 +08:00
parent 936094211b
commit 6dc4e2b943
119 changed files with 7427 additions and 357 deletions

View File

@@ -0,0 +1,6 @@
"""
Personalized Reco个性化推荐功能模块集合。
该目录用于承载推荐引擎与其子模块(数据访问、打分、重排、可观测等)。
"""

View File

@@ -0,0 +1,8 @@
"""
Content Repository候选查询与数据访问层
说明:
- 本模块为推荐引擎提供可注入的数据访问接口(与 ORM/SQL 解耦)。
- 负责将 DB 存储形态规范化为上层稳定的 ContentProfile 结构。
"""

View File

@@ -0,0 +1,36 @@
from __future__ import annotations
from typing import Protocol
from app.features.personalized_reco.content_repository.types import ContentProfileDTO
class ContentRepository(Protocol):
"""
推荐引擎依赖的内容数据访问抽象接口(用于解耦 ORM/SQL
"""
async def fetch_candidates(
self,
*,
scene: str,
user_profile: object,
fallback_level: int,
limit: int,
locale: str,
exclude_content_ids: list[int] | None = None,
) -> list[ContentProfileDTO]:
"""
按场景与用户画像拉取候选内容画像(用于候选池)。
"""
async def fetch_contents_by_ids(
self,
*,
content_ids: list[int],
locale: str,
) -> list[ContentProfileDTO]:
"""
按 content_id 批量获取内容画像(去重、按输入顺序返回;缺语言/缺记录的 id 跳过)。
"""

View File

@@ -0,0 +1,130 @@
from __future__ import annotations
from typing import Any
from app.features.personalized_reco.content_repository.types import Locale, normalize_locale
CONTEXT_KEYS: tuple[str, ...] = ("family", "work", "relationship", "friends", "health")
NEED_KEYS: tuple[str, ...] = (
"emotional_support",
"parenting_pressure",
"self_worth",
"anxiety_relief",
"rest_balance",
)
def _normalize_discrete_score(v: Any, *, default: float = 0.5) -> float:
"""
将 suitability 的离散值规范化为 0/0.5/1。
非法值一律兜底 default默认 0.5)。
"""
try:
if v in (0, 0.0):
return 0.0
if v in (0.5,):
return 0.5
if v in (1, 1.0):
return 1.0
# 允许字符串形式的 "0"/"0.5"/"1"
if isinstance(v, str):
s = v.strip()
if s == "0":
return 0.0
if s == "0.5":
return 0.5
if s == "1":
return 1.0
except Exception:
return default
return default
def normalize_suitability(raw: Any, *, keys: tuple[str, ...]) -> dict[str, float]:
"""
解析 suitability JSON缺失时补齐全 0.5。
raw 期望为 dict否则视为缺失。
"""
data: dict[str, Any] = raw if isinstance(raw, dict) else {}
return {k: _normalize_discrete_score(data.get(k), default=0.5) for k in keys}
def normalize_review_confidence(raw: Any) -> float:
"""
review_confidence 缺失/NULL 时兜底 0.7。
"""
try:
if raw is None:
return 0.7
v = float(raw)
if 0.0 <= v <= 1.0:
return v
except Exception:
pass
return 0.7
def normalize_personalization_power(raw: Any) -> float:
"""
DB 约定存 0/5/10读取层输出 0/0.5/1。
"""
try:
if raw is None:
return 0.0
v = int(raw)
if v == 0:
return 0.0
if v == 5:
return 0.5
if v == 10:
return 1.0
except Exception:
pass
return 0.0
_RISK_FLAG_MAP: dict[str, str] = {
"block_stage_unknown": "unsafe_for_stage_unknown",
"block_stage_parenting": "unsafe_for_stage_parenting",
"block_emotion_low": "unsafe_for_emotion_low",
"block_health_sensitive": "block_health_medical",
}
def normalize_risk_flags(raw_flags: list[str] | None) -> list[str]:
"""
risk_flags 旧→新映射、去重、稳定排序(字典序)。
"""
flags = raw_flags or []
mapped: set[str] = set()
for f in flags:
if not f:
continue
name = _RISK_FLAG_MAP.get(f, f)
mapped.add(name)
return sorted(mapped)
def pick_text(*, text_en: str | None, text_tc: str | None, locale: str) -> str | None:
"""
按 locale 选择输出文案文本。
当前仅支持 EN/TC且不允许语言回退
- locale=en*:必须使用 text_en
- locale=tc/zh-TW/zh-HK必须使用 text_tc
"""
loc: Locale = normalize_locale(locale)
if loc == "en":
return text_en if text_en else None
# loc == "tc"
return text_tc if text_tc else None

View File

@@ -0,0 +1,265 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass
from typing import Any, Iterable
from sqlalchemy import Select, and_, desc, not_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models.content import Content
from app.db.models.content_profile import ContentProfile
from app.db.models.content_risk_flag import ContentRiskFlag
from app.features.personalized_reco.content_repository.interface import ContentRepository
from app.features.personalized_reco.content_repository.normalization import (
CONTEXT_KEYS,
NEED_KEYS,
normalize_personalization_power,
normalize_review_confidence,
normalize_risk_flags,
normalize_suitability,
pick_text,
)
from app.features.personalized_reco.content_repository.types import ContentProfileDTO
@dataclass(frozen=True)
class _UserSignals:
"""
从 user_profile 中提取 repository 级别需要的最小信号。
注意更复杂的规则Hard Filter/Scoring/Rerank不在本层处理。
"""
missing_need: bool
missing_context: bool
missing_emotion: bool
stage: str | None # expecting/parenting/unknown/general/None
def _bool(v: Any) -> bool:
return bool(v)
def _extract_user_signals(user_profile: object) -> _UserSignals:
"""
兼容 pydantic model / dict / 其他对象的最小字段读取。
"""
def _get(obj: Any, key: str, default: Any = None) -> Any:
if obj is None:
return default
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
need = _get(user_profile, "need", {}) or {}
context = _get(user_profile, "context", {}) or {}
emotion_score = _get(user_profile, "emotion_score", None)
missing_need = len(need) == 0
missing_context = len(context) == 0
missing_emotion = emotion_score is None
# stage: from user_profile.stage (one-hot)
stage_obj = _get(user_profile, "stage", None)
stage: str | None = None
if stage_obj is not None:
expecting = _get(stage_obj, "expecting", None)
parenting = _get(stage_obj, "parenting", None)
unknown = _get(stage_obj, "unknown", None)
if _bool(expecting):
stage = "expecting"
elif _bool(parenting):
stage = "parenting"
elif _bool(unknown):
stage = "unknown"
return _UserSignals(
missing_need=missing_need,
missing_context=missing_context,
missing_emotion=missing_emotion,
stage=stage,
)
def _dedupe_preserve_order(ids: Iterable[int]) -> list[int]:
seen: set[int] = set()
out: list[int] = []
for i in ids:
if i in seen:
continue
seen.add(i)
out.append(i)
return out
class SqlAlchemyContentRepository(ContentRepository):
"""
基于 SQLAlchemy AsyncSession 的 ContentRepository 实现。
"""
def __init__(self, session: AsyncSession):
self._session = session
async def fetch_contents_by_ids(self, *, content_ids: list[int], locale: str) -> list[ContentProfileDTO]:
"""
- 输入去重
- 输出顺序与输入一致(按首次出现顺序)
- 缺记录或缺目标语言文本:跳过
- 不产生 N+1主体+画像一次flags 一次)
"""
unique_ids = _dedupe_preserve_order(content_ids)
if not unique_ids:
return []
# locale 文本存在性过滤(不允许语言回退)
# en -> 必须 text_entc -> 必须 text_tc
# 过滤在 DB 层做,避免后续组装无意义
from app.features.personalized_reco.content_repository.types import normalize_locale
loc = normalize_locale(locale)
text_filter = Content.text_en.is_not(None) if loc == "en" else Content.text_tc.is_not(None)
stmt: Select = (
select(Content, ContentProfile)
.join(ContentProfile, Content.content_id == ContentProfile.content_id)
.where(and_(Content.content_id.in_(unique_ids), text_filter))
)
rows = (await self._session.execute(stmt)).all()
if not rows:
return []
# 先组装主体+画像,后续再补 risk_flags
by_id: dict[int, dict[str, Any]] = {}
valid_ids: list[int] = []
for content, profile in rows:
cid = int(content.content_id)
text = pick_text(text_en=content.text_en, text_tc=content.text_tc, locale=locale)
if not text:
continue
by_id[cid] = {
"content": content,
"profile": profile,
"text": text,
}
valid_ids.append(cid)
if not by_id:
return []
# 批量取 flags避免 join 行膨胀)
flags_stmt = select(ContentRiskFlag.content_id, ContentRiskFlag.flag).where(
ContentRiskFlag.content_id.in_(list(by_id.keys()))
)
flags_rows = (await self._session.execute(flags_stmt)).all()
flags_map: dict[int, list[str]] = defaultdict(list)
for cid, flag in flags_rows:
flags_map[int(cid)].append(str(flag))
result_by_id: dict[int, ContentProfileDTO] = {}
for cid, payload in by_id.items():
content: Content = payload["content"]
profile: ContentProfile = payload["profile"]
text: str = payload["text"]
dto = ContentProfileDTO(
content_id=cid,
text=text,
stage=profile.stage, # type: ignore[arg-type]
emotion_score=float(profile.emotion_score) if profile.emotion_score is not None else None,
context_suitability=normalize_suitability(profile.context_suitability_json, keys=CONTEXT_KEYS),
need_suitability=normalize_suitability(profile.need_suitability_json, keys=NEED_KEYS),
personalization_power=normalize_personalization_power(profile.personalization_power),
risk_flags=normalize_risk_flags(flags_map.get(cid)),
author_id=content.author_id,
template_id=content.template_id,
review_confidence=normalize_review_confidence(profile.review_confidence),
)
result_by_id[cid] = dto
# 按输入顺序返回(跳过缺失/被过滤的)
out: list[ContentProfileDTO] = []
for cid in unique_ids:
dto = result_by_id.get(cid)
if dto is not None:
out.append(dto)
return out
async def fetch_candidates(
self,
*,
scene: str,
user_profile: object,
fallback_level: int,
limit: int,
locale: str,
exclude_content_ids: list[int] | None = None,
) -> list[ContentProfileDTO]:
"""
两段式候选召回:
1) 先查候选 content_id 列表含粗过滤、locale 过滤、limit*multiplier
2) 再批量补全字段(复用 fetch_contents_by_ids
"""
if limit <= 0:
return []
signals = _extract_user_signals(user_profile)
effective_fallback = int(fallback_level)
if signals.missing_need or signals.missing_context or signals.missing_emotion:
effective_fallback = max(effective_fallback, 1)
# locale 文本存在性过滤(不允许语言回退)
from app.features.personalized_reco.content_repository.types import normalize_locale
loc = normalize_locale(locale)
text_filter = Content.text_en.is_not(None) if loc == "en" else Content.text_tc.is_not(None)
filters: list[Any] = [text_filter]
if exclude_content_ids:
filters.append(not_(Content.content_id.in_(exclude_content_ids)))
# fallback 约束repository 只做“降级约束”,不做 hard filter
if effective_fallback >= 1:
# personalization_power <= 5 代表 <= 0.5
filters.append(ContentProfile.personalization_power <= 5)
if effective_fallback >= 2:
filters.append(ContentProfile.personalization_power == 0)
filters.append(ContentProfile.stage == "general")
if effective_fallback >= 3:
filters.append(ContentProfile.is_safe_pool.is_(True))
filters.append(ContentProfile.personalization_power == 0)
filters.append(ContentProfile.stage == "general")
# stage 粗过滤(仅 L0/L1 才做“用户阶段 + general”L2/L3 已强制 general
if effective_fallback < 2:
user_stage = signals.stage
if user_stage in {"expecting", "parenting"}:
filters.append(ContentProfile.stage.in_([user_stage, "general"]))
else:
# unknown 或无法判定:仅取 general避免误推
filters.append(ContentProfile.stage == "general")
multiplier = 5
raw_limit = max(limit * multiplier, limit)
stmt_ids = (
select(Content.content_id)
.join(ContentProfile, Content.content_id == ContentProfile.content_id)
.where(and_(*filters))
.order_by(desc(ContentProfile.updated_at))
.limit(raw_limit)
)
candidate_ids_rows = (await self._session.execute(stmt_ids)).scalars().all()
candidate_ids = [int(x) for x in candidate_ids_rows]
if not candidate_ids:
return []
# 复用按 ID 批量补全(会再次做 locale 过滤,但成本可接受,且可保证一致行为)
items = await self.fetch_contents_by_ids(content_ids=candidate_ids, locale=locale)
return items[:limit]

View File

@@ -0,0 +1,64 @@
from __future__ import annotations
from typing import Literal, Optional
from pydantic import BaseModel, Field
# 当前阶段仅支持 EN / TC繁体中文
Locale = Literal["en", "tc"]
def normalize_locale(locale: str) -> Locale:
"""
将客户端传入的 locale 归一化为内部枚举(仅 EN / TC
约定:
- 任何以 "en" 开头的 locale 归一化为 "en"(例如 en、en-US
- "tc"/"zh-TW"/"zh-HK" 归一化为 "tc"
- 其他 locale 视为不支持
"""
raw = (locale or "").strip()
if not raw:
raise ValueError("locale 不能为空(当前仅支持 en/tc")
low = raw.lower()
if low.startswith("en"):
return "en"
if low in {"tc", "zh-tw", "zh-hk", "zh_tw", "zh_hk"}:
return "tc"
raise ValueError(f"不支持的 locale{locale!r}(当前仅支持 en/tc")
ContentStage = Literal["general", "expecting", "parenting", "unknown"]
class ContentProfileDTO(BaseModel):
"""
推荐模块消费的内容画像(稳定字段契约)。
注意:
- text 已按 locale 选择,不允许语言回退(缺语言文本的内容不返回)
- emotion_score 为 None 表示 general
- personalization_power 对上统一为 0/0.5/1
- review_confidence 缺失时兜底 0.7
"""
content_id: int
text: str
stage: ContentStage
emotion_score: Optional[float] = None
context_suitability: dict[str, float] = Field(default_factory=dict)
need_suitability: dict[str, float] = Field(default_factory=dict)
personalization_power: float
risk_flags: list[str] = Field(default_factory=list)
# 可选字段
author_id: Optional[str] = None
template_id: Optional[str] = None
review_confidence: float = 0.7

View File

@@ -0,0 +1,20 @@
"""
个性化推荐Observability 子模块(可观测性与打点载荷)
说明:
- 只负责统一 `RecoMeta` 结构与构建builder不负责埋点 SDK/落库/上报实现。
- `RecoMeta` 需要同时被 `reco-engine` 与 `integration-api-worker` 使用。
"""
from .builder import RecoMetaBuilder
from .types import MissingFields, RecoMeta
from .utils import compute_empty_reason, compute_missing_fields
__all__ = [
"MissingFields",
"RecoMeta",
"RecoMetaBuilder",
"compute_empty_reason",
"compute_missing_fields",
]

View File

@@ -0,0 +1,136 @@
from __future__ import annotations
import logging
from datetime import datetime
from typing import Any, Optional
from app.features.personalized_reco.observability.types import MissingFields, RecoMeta, Scene
from app.features.personalized_reco.observability.utils import compute_empty_reason, compute_missing_fields
logger = logging.getLogger(__name__)
def _non_negative_int(value: Any, *, default: int = 0) -> int:
try:
n = int(value)
except Exception:
return int(default)
return max(0, int(n))
class RecoMetaBuilder:
"""
在推荐 pipeline 中逐阶段填充 RecoMeta避免“散落字段/散落日志”。
说明V1
- set 调用允许任意顺序build 时会做防御式兜底与单调性修正
- 单调性约束raw >= after_hard_filter >= after_dedup >= after_freqcap >= served_k
"""
def __init__(self, *, scene: Scene, user_profile: object, k: int, now: Optional[datetime] = None) -> None:
self.scene: Scene = scene
self.user_profile = user_profile
self.k = _non_negative_int(k, default=0)
self.now = now
self._raw: Optional[int] = None
self._after_hard: Optional[int] = None
self._after_dedup: Optional[int] = None
self._after_freqcap: Optional[int] = None
self._served_k: Optional[int] = None
self._fallback_level_final: Optional[int] = None
self._risk_filtered_count_by_flag: dict[str, int] = {}
self._freqcap_filtered_counts: dict[str, int] = {}
self._config_snapshot: dict[str, Any] = {}
def set_candidate_pool_size_raw(self, n: Any) -> "RecoMetaBuilder":
self._raw = _non_negative_int(n)
return self
def set_after_hard_filter(self, n: Any, *, risk_filtered_count_by_flag: Optional[dict[str, Any]] = None) -> "RecoMetaBuilder":
self._after_hard = _non_negative_int(n)
if risk_filtered_count_by_flag:
self._risk_filtered_count_by_flag = {str(k): _non_negative_int(v) for k, v in risk_filtered_count_by_flag.items()}
return self
def set_after_dedup(self, n: Any) -> "RecoMetaBuilder":
self._after_dedup = _non_negative_int(n)
return self
def set_after_freqcap(self, n: Any, *, freqcap_filtered_counts: Optional[dict[str, Any]] = None) -> "RecoMetaBuilder":
self._after_freqcap = _non_negative_int(n)
if freqcap_filtered_counts:
self._freqcap_filtered_counts = {str(k): _non_negative_int(v) for k, v in freqcap_filtered_counts.items()}
return self
def set_fallback_level_final(self, level: Any, *, reason: Optional[str] = None) -> "RecoMetaBuilder":
# reason 预留V1 先不入 meta可放入 config_snapshot 或后续字段)
self._fallback_level_final = _non_negative_int(level, default=0)
if reason:
self._config_snapshot.setdefault("fallback_trigger_reason", str(reason))
return self
def set_served_k(self, n: Any) -> "RecoMetaBuilder":
self._served_k = _non_negative_int(n)
return self
def set_config_snapshot(self, snapshot: dict[str, Any]) -> "RecoMetaBuilder":
self._config_snapshot = dict(snapshot or {})
return self
def build(self) -> RecoMeta:
missing: MissingFields = compute_missing_fields(self.user_profile)
conf_u = getattr(self.user_profile, "profile_confidence", 1.0)
try:
conf_u_f = float(conf_u)
except Exception:
conf_u_f = 1.0
if conf_u_f != conf_u_f:
conf_u_f = 1.0
raw = self._raw if self._raw is not None else 0
after_hard = self._after_hard if self._after_hard is not None else raw
after_dedup = self._after_dedup if self._after_dedup is not None else after_hard
after_freqcap = self._after_freqcap if self._after_freqcap is not None else after_dedup
served_k = self._served_k if self._served_k is not None else 0
# 防御式单调性修正(以最保守值输出)
if after_hard > raw:
logger.debug("after_hard_filter(%s) > raw(%s),已修正为 raw", after_hard, raw)
after_hard = raw
if after_dedup > after_hard:
logger.debug("after_dedup(%s) > after_hard_filter(%s),已修正为 after_hard_filter", after_dedup, after_hard)
after_dedup = after_hard
if after_freqcap > after_dedup:
logger.debug("after_freqcap(%s) > after_dedup(%s),已修正为 after_dedup", after_freqcap, after_dedup)
after_freqcap = after_dedup
if served_k > after_freqcap:
logger.debug("served_k(%s) > after_freqcap(%s),已修正为 after_freqcap", served_k, after_freqcap)
served_k = after_freqcap
fallback_level_final = self._fallback_level_final if self._fallback_level_final is not None else 0
empty_reason = compute_empty_reason(
served_k=served_k,
candidate_pool_size_raw=raw,
candidate_pool_size_after_hard_filter=after_hard,
candidate_pool_size_after_freqcap=after_freqcap,
)
return RecoMeta(
scene=self.scene,
candidate_pool_size_raw=int(raw),
candidate_pool_size_after_hard_filter=int(after_hard),
candidate_pool_size_after_dedup=int(after_dedup),
candidate_pool_size_after_freqcap=int(after_freqcap),
fallback_level_final=int(fallback_level_final),
served_k=int(served_k),
empty_reason=empty_reason,
conf_U=float(conf_u_f),
missing_fields=missing,
risk_filtered_count_by_flag=dict(self._risk_filtered_count_by_flag),
freqcap_filtered_counts=dict(self._freqcap_filtered_counts),
config_snapshot=dict(self._config_snapshot),
)

View File

@@ -0,0 +1,51 @@
from __future__ import annotations
from typing import Any, Literal, Optional
from pydantic import BaseModel, Field
Scene = Literal["feed", "push", "widget"]
EmptyReason = Literal["hard_filter_all", "freqcap_all", "pool_empty", "unknown"]
class MissingFields(BaseModel):
"""
画像字段缺失情况(布尔结构)。
"""
need: bool = False
context: bool = False
emotion: bool = False
class RecoMeta(BaseModel):
"""
推荐模块统一可观测载荷(返回给调用方;调用方负责上报/落库/打点)。
"""
scene: Scene
candidate_pool_size_raw: int = 0
candidate_pool_size_after_hard_filter: int = 0
candidate_pool_size_after_dedup: int = 0
candidate_pool_size_after_freqcap: int = 0
fallback_level_final: int = 0
served_k: int = 0
# served_k=0 时必填served_k>0 时建议为 None
empty_reason: Optional[EmptyReason] = None
conf_U: float = 1.0
missing_fields: MissingFields = Field(default_factory=MissingFields)
# 可选Hard Filter 风险命中统计(按 flag 聚合)
risk_filtered_count_by_flag: dict[str, int] = Field(default_factory=dict)
# 可选Freqcap 过滤统计sentence/author/template
freqcap_filtered_counts: dict[str, int] = Field(default_factory=dict)
# 可选调参快照V1 可先只在内部事件使用)
config_snapshot: dict[str, Any] = Field(default_factory=dict)

View File

@@ -0,0 +1,61 @@
from __future__ import annotations
from typing import Optional
from app.features.personalized_reco.observability.types import EmptyReason, MissingFields
def compute_missing_fields(user_profile: object) -> MissingFields:
"""
判定用户画像缺失字段(对齐算法规则 V1.2 口径)。
规则:
- needuser_profile.need 为空对象 {} 或不存在
- contextuser_profile.context 为空对象 {} 或不存在
- emotionuser_profile.emotion_score 为 None 或不存在
"""
need = getattr(user_profile, "need", None)
context = getattr(user_profile, "context", None)
emotion_score = getattr(user_profile, "emotion_score", None)
need_missing = not bool(need)
context_missing = not bool(context)
emotion_missing = emotion_score is None
return MissingFields(need=need_missing, context=context_missing, emotion=emotion_missing)
def compute_empty_reason(
*,
served_k: int,
candidate_pool_size_raw: int,
candidate_pool_size_after_hard_filter: int,
candidate_pool_size_after_freqcap: int,
) -> Optional[EmptyReason]:
"""
判定 empty_reasonserved_k=0 必填)。
规则(对齐 plan
- served_k>0 -> None
- raw==0 -> pool_empty
- raw>0 且 after_hard_filter==0 -> hard_filter_all
- after_freqcap==0 -> freqcap_all
- 其他 -> unknown
"""
if int(served_k) > 0:
return None
raw = int(candidate_pool_size_raw)
after_hard = int(candidate_pool_size_after_hard_filter)
after_freqcap = int(candidate_pool_size_after_freqcap)
if raw == 0:
return "pool_empty"
if raw > 0 and after_hard == 0:
return "hard_filter_all"
if after_freqcap == 0:
return "freqcap_all"
return "unknown"

View File

@@ -0,0 +1,11 @@
"""
Reco Engine推荐引擎编排
该模块负责将候选拉取、硬过滤、软打分、重排/频控、回退梯度串成一个稳定 Pipeline
并输出统一结构items + meta可观测字段
"""
from app.features.personalized_reco.reco_engine.orchestrator import recommend
__all__ = ["recommend"]

View File

@@ -0,0 +1,14 @@
from __future__ import annotations
from app.features.personalized_reco.reco_engine.types import RecoEngineConfig, Scene
def get_default_engine_config(scene: Scene) -> RecoEngineConfig:
"""
获取推荐引擎默认配置(返回副本,避免被意外修改)。
"""
# V1三种场景目前共用一套默认值保留 scene 参数便于后续按场景拆分
base = RecoEngineConfig()
return RecoEngineConfig.model_validate(base.model_dump())

View File

@@ -0,0 +1,128 @@
from __future__ import annotations
from collections import defaultdict
from typing import Any, Iterable, Optional
from app.features.personalized_reco.content_repository.types import ContentProfileDTO
from app.features.personalized_reco.reco_engine.types import HardFilterResult, RecoConstraints, Scene
def _user_stage_key(user_profile: object) -> str:
"""
从 user_profile.stage(one-hot) 提取用户阶段。
约定unknown 通常必填,但这里做防御。
"""
stage_obj = getattr(user_profile, "stage", None)
if stage_obj is None:
return "unknown"
if getattr(stage_obj, "expecting", 0) == 1:
return "expecting"
if getattr(stage_obj, "parenting", 0) == 1:
return "parenting"
return "unknown"
def _user_emotion_score(user_profile: object) -> Optional[float]:
v = getattr(user_profile, "emotion_score", None)
if v is None:
return None
try:
f = float(v)
except Exception:
return None
if f != f:
return None
return f
def _count_hits(counter: dict[str, int], hits: Iterable[str]) -> None:
for h in hits:
counter[str(h)] += 1
def hard_filter(
*,
scene: Scene,
user_profile: object,
candidates: list[ContentProfileDTO],
constraints: Optional[RecoConstraints] = None,
) -> HardFilterResult:
"""
Hard Filter硬过滤
V1仅实现硬规则集合不做软惩罚不做扩展 hard_rules
"""
cons = constraints or RecoConstraints()
exclude_author_ids = set([a for a in (cons.exclude_author_ids or []) if a is not None and str(a).strip() != ""])
exclude_template_ids = set([t for t in (cons.exclude_template_ids or []) if t is not None and str(t).strip() != ""])
exclude_content_ids = set([int(x) for x in (cons.exclude_content_ids or []) if x is not None])
u_stage = _user_stage_key(user_profile)
u_emotion = _user_emotion_score(user_profile)
emotion_low = u_emotion is not None and float(u_emotion) <= 0.2
kept: list[ContentProfileDTO] = []
removed_count = 0
# 统计:按命中 key 聚合计数risk_flags 直接用 flag 字符串;跨维度/约束用 rule:* / constraint:* 前缀)
hit_counts: dict[str, int] = defaultdict(int)
hits_by_content_id: dict[int, list[str]] = {}
for c in candidates or []:
cid = int(c.content_id)
hits: list[str] = []
# 约束:按 content_id/author_id/template_id 排除(视为硬过滤)
if cid in exclude_content_ids:
hits.append("constraint:exclude_content_id")
if c.author_id and c.author_id in exclude_author_ids:
hits.append("constraint:exclude_author_id")
if c.template_id and c.template_id in exclude_template_ids:
hits.append("constraint:exclude_template_id")
flags = set([str(x) for x in (c.risk_flags or []) if x is not None and str(x).strip() != ""])
# 全场景必挡
if "block_health_medical" in flags:
hits.append("block_health_medical")
# 与用户阶段相关
if u_stage == "unknown" and "unsafe_for_stage_unknown" in flags:
hits.append("unsafe_for_stage_unknown")
if u_stage == "parenting" and "unsafe_for_stage_parenting" in flags:
hits.append("unsafe_for_stage_parenting")
# 与用户情绪相关
if emotion_low and "unsafe_for_emotion_low" in flags:
hits.append("unsafe_for_emotion_low")
# 跨维度规则unknown stage + parenting_pressure 强命中 + 高个性化
if u_stage == "unknown":
try:
need_val = float(c.need_suitability.get("parenting_pressure", 0.0))
except Exception:
need_val = 0.0
if need_val >= 1.0 and float(getattr(c, "personalization_power", 0.0)) >= 1.0:
hits.append("rule:unknown_stage_parenting_pressure_power1")
if hits:
removed_count += 1
# 单条去重后再计数,避免同 key 重复
uniq_hits = sorted(set(hits))
hits_by_content_id[cid] = uniq_hits
_count_hits(hit_counts, uniq_hits)
continue
hits_by_content_id[cid] = []
kept.append(c)
return HardFilterResult(
kept_items=kept,
removed_count=int(removed_count),
risk_filtered_count_by_flag=dict(hit_counts),
hits_by_content_id=hits_by_content_id,
)

View File

@@ -0,0 +1,396 @@
from __future__ import annotations
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any, Optional
from app.features.personalized_reco.content_repository.interface import ContentRepository
from app.features.personalized_reco.content_repository.types import ContentProfileDTO, normalize_locale
from app.features.personalized_reco.observability.builder import RecoMetaBuilder
from app.features.personalized_reco.reco_engine.defaults import get_default_engine_config
from app.features.personalized_reco.reco_engine.hard_filter import hard_filter
from app.features.personalized_reco.reco_engine.types import RecoConstraints, RecoEngineConfig, RecoEngineResult, RecommendedItem, Scene
from app.features.personalized_reco.reco_engine.utils import (
clamp_personalization_power,
merge_exclude_ids,
normalize_or_default_locale,
)
from app.features.personalized_reco.rerank_freqcap.rerank import rerank_and_freqcap
from app.features.personalized_reco.rerank_freqcap.types import ScoredCandidate
from app.features.personalized_reco.scoring.defaults import get_default_config as get_default_score_config
from app.features.personalized_reco.scoring.score import score_content
logger = logging.getLogger(__name__)
def _safe_int(value: Any, *, default: int = 0) -> int:
try:
n = int(value)
except Exception:
return int(default)
return int(n)
def _light_score_summary(score_result: Any) -> dict[str, Any]:
"""
轻量 explanations只保留少量关键字段避免 payload 过大。
"""
bd = getattr(score_result, "breakdown", None)
if bd is None:
return {}
def _get(name: str) -> Optional[float]:
v = getattr(bd, name, None)
if v is None:
return None
try:
f = float(v)
except Exception:
return None
if f != f:
return None
return f
out: dict[str, Any] = {
"missing_fields": list(getattr(bd, "missing_fields", []) or []),
"S_core": _get("S_core"),
"S_personal": _get("S_personal"),
"P_uncertainty": _get("P_uncertainty"),
"P_risk": _get("P_risk"),
"P_widget_emotion_out_of_range": _get("P_widget_emotion_out_of_range"),
}
# 删除 None减少噪音
return {k: v for k, v in out.items() if v is not None and v != []}
def _apply_fallback_level_to_content(content: ContentProfileDTO, *, fallback_level: int) -> ContentProfileDTO:
"""
对内容做防御式一致性处理(与回退梯度一致)。
"""
p2 = clamp_personalization_power(content.personalization_power, fallback_level=fallback_level)
if p2 == content.personalization_power:
return content
return content.model_copy(update={"personalization_power": float(p2)})
def _merge_counter(dst: dict[str, int], src: dict[str, Any]) -> None:
for k, v in (src or {}).items():
try:
n = int(v)
except Exception:
n = 0
dst[str(k)] = int(dst.get(str(k), 0)) + max(0, int(n))
async def recommend(
*,
repo: ContentRepository,
scene: Scene,
user_profile: object,
already_recommended_ids: list[Any],
touched_or_viewed_ids: list[Any],
k: int,
now: datetime,
locale: Optional[str] = None,
constraints: Optional[RecoConstraints] = None,
config: Optional[RecoEngineConfig] = None,
) -> RecoEngineResult:
"""
Reco Engine 主入口:编排候选→过滤→打分→重排→回退,并输出 items + meta。
"""
cfg = config or get_default_engine_config(scene)
cons = constraints or RecoConstraints()
k_i = max(0, _safe_int(k, default=0))
meta_builder = RecoMetaBuilder(scene=scene, user_profile=user_profile, k=k_i, now=now)
if k_i <= 0:
meta_builder.set_candidate_pool_size_raw(0).set_after_hard_filter(0).set_after_dedup(0).set_after_freqcap(0).set_served_k(0).set_fallback_level_final(0)
meta_builder.set_config_snapshot({"engine_note": "k<=0直接返回空结果"})
return RecoEngineResult(items=[], meta=meta_builder.build())
# locale默认 en严格校验仅支持 en/tc
raw_locale = normalize_or_default_locale(locale)
try:
effective_locale = normalize_locale(raw_locale)
except Exception as e:
meta_builder.set_config_snapshot({"error": str(e), "stage": "normalize_locale", "locale": raw_locale})
meta_builder.set_candidate_pool_size_raw(0).set_after_hard_filter(0).set_after_dedup(0).set_after_freqcap(0).set_served_k(0).set_fallback_level_final(0)
return RecoEngineResult(items=[], meta=meta_builder.build())
# 聚合统计(跨回退层级累加,确保 meta 单调性成立)
raw_total = 0
after_hard_total = 0
after_dedup_total = 0
after_freqcap_total = 0
risk_counts_total: dict[str, int] = defaultdict(int)
freqcap_counts_total: dict[str, int] = defaultdict(int)
fallback_trace: list[dict[str, Any]] = []
selected: list[ScoredCandidate] = []
selected_level_by_id: dict[int, int] = {}
last_fallback_level = 0
last_reason = None
for level in [0, 1, 2, 3]:
last_fallback_level = int(level)
k_remaining = max(0, k_i - len(selected))
if k_remaining <= 0:
break
# Feed允许不足且不补齐时拿到任何结果就停止
if scene == "feed" and cfg.feed_allow_partial and (not cfg.feed_fill_with_fallback) and len(selected) > 0:
break
# exclude_idsalready/touched + constraints.exclude + 已选内容(避免跨层重复)
exclude_ids = merge_exclude_ids(
already_recommended_ids=list(already_recommended_ids or []) + [int(x.content_id) for x in selected],
touched_or_viewed_ids=list(touched_or_viewed_ids or []),
extra_exclude_content_ids=list(cons.exclude_content_ids or []),
)
multiplier = int(cfg.candidate_multiplier_feed if scene == "feed" else cfg.candidate_multiplier_push_widget)
base_limit = max(int(cfg.min_candidates_per_level), int(k_remaining) * max(1, int(multiplier)))
if cons.max_candidates_limit is not None and int(cons.max_candidates_limit) > 0:
limit = min(base_limit, int(cons.max_candidates_limit))
else:
limit = base_limit
# 1) Candidate
try:
cands = await repo.fetch_candidates(
scene=scene,
user_profile=user_profile,
fallback_level=int(level),
limit=int(limit),
locale=str(effective_locale),
exclude_content_ids=exclude_ids,
)
except Exception as e:
logger.exception("fetch_candidates 失败:%s", e)
last_reason = "error:fetch_candidates"
fallback_trace.append(
{
"level": int(level),
"raw": 0,
"after_hard": 0,
"after_dedup": 0,
"after_freqcap": 0,
"served_total": len(selected),
"error": str(e),
}
)
continue
raw_total += len(cands)
if not cands:
last_reason = "pool_empty"
fallback_trace.append(
{
"level": int(level),
"raw": 0,
"after_hard": 0,
"after_dedup": 0,
"after_freqcap": 0,
"served_total": len(selected),
"reason": "pool_empty",
}
)
continue
# 2) Hard Filter
hf = hard_filter(scene=scene, user_profile=user_profile, candidates=cands, constraints=cons)
kept = [x for x in hf.kept_items if isinstance(x, ContentProfileDTO)]
after_hard_total += len(kept)
_merge_counter(risk_counts_total, hf.risk_filtered_count_by_flag)
if not kept:
last_reason = "hard_filter_all"
fallback_trace.append(
{
"level": int(level),
"raw": len(cands),
"after_hard": 0,
"after_dedup": 0,
"after_freqcap": 0,
"served_total": len(selected),
"reason": "hard_filter_all",
}
)
continue
# 3) Soft Scoring
score_cfg = get_default_score_config(scene)
if scene == "push":
# Push强制启用不确定性惩罚与 spec 对齐)
score_cfg = score_cfg.model_copy(update={"enable_uncertainty_penalty": True})
scored: list[ScoredCandidate] = []
for c in kept:
c2 = _apply_fallback_level_to_content(c, fallback_level=int(level))
try:
s = score_content(scene=scene, user_profile=user_profile, content_profile=c2, config=score_cfg, pass_filters=True, now=now)
except Exception as e:
# 单条异常不影响整体
logger.exception("score_content 失败 content_id=%s%s", getattr(c2, "content_id", None), e)
continue
cid = int(c2.content_id)
hits = hf.hits_by_content_id.get(cid, [])
extra: dict[str, Any] = {
"text": c2.text,
"fallback_level_used": int(level),
}
if cfg.enable_explanations:
extra["hard_filter_hits"] = hits
extra["score_summary"] = _light_score_summary(s)
scored.append(
ScoredCandidate(
content_id=cid,
final_score=float(getattr(s, "final_score", 0.0)),
author_id=c2.author_id,
template_id=c2.template_id,
content_profile=c2,
extra=extra,
)
)
if not scored:
last_reason = "empty_after_scoring"
fallback_trace.append(
{
"level": int(level),
"raw": len(cands),
"after_hard": len(kept),
"after_dedup": 0,
"after_freqcap": 0,
"served_total": len(selected),
"reason": "empty_after_scoring",
}
)
continue
# 4) Rerank/Freqcap
try:
rer = rerank_and_freqcap(
scene=scene,
scored_candidates=scored,
already_recommended_ids=list(already_recommended_ids or []) + [int(x.content_id) for x in selected],
touched_or_viewed_ids=list(touched_or_viewed_ids or []),
k=int(k_remaining),
recent_author_ids=cons.recent_author_ids,
recent_template_ids=cons.recent_template_ids,
)
except Exception as e:
logger.exception("rerank_and_freqcap 失败:%s", e)
last_reason = "error:rerank_and_freqcap"
fallback_trace.append(
{
"level": int(level),
"raw": len(cands),
"after_hard": len(kept),
"after_dedup": 0,
"after_freqcap": 0,
"served_total": len(selected),
"error": str(e),
}
)
continue
after_dedup_total += int(rer.meta.candidate_pool_size_after_dedup)
after_freqcap_total += int(rer.meta.candidate_pool_size_after_freqcap)
_merge_counter(freqcap_counts_total, rer.meta.freqcap_filtered_counts)
served_level = list(rer.ranked_items or [])[:k_remaining]
if not served_level:
last_reason = "freqcap_all"
fallback_trace.append(
{
"level": int(level),
"raw": len(cands),
"after_hard": len(kept),
"after_dedup": int(rer.meta.candidate_pool_size_after_dedup),
"after_freqcap": int(rer.meta.candidate_pool_size_after_freqcap),
"served_total": len(selected),
"reason": "freqcap_all",
}
)
continue
for it in served_level:
cid = int(it.content_id)
selected.append(it)
selected_level_by_id[cid] = int(level)
last_reason = None
fallback_trace.append(
{
"level": int(level),
"raw": len(cands),
"after_hard": len(kept),
"after_dedup": int(rer.meta.candidate_pool_size_after_dedup),
"after_freqcap": int(rer.meta.candidate_pool_size_after_freqcap),
"served_total": len(selected),
"served_added": len(served_level),
}
)
if len(selected) >= k_i:
break
# 组装输出 items按 selected 顺序)
items: list[RecommendedItem] = []
for c in selected[:k_i]:
cid = int(c.content_id)
text = ""
if isinstance(c.extra, dict):
text = str(c.extra.get("text") or "")
explanations = None
if cfg.enable_explanations and isinstance(c.extra, dict):
explanations = {
"fallback_level_used": c.extra.get("fallback_level_used"),
"hard_filter_hits": c.extra.get("hard_filter_hits"),
"score_summary": c.extra.get("score_summary"),
}
items.append(
RecommendedItem(
content_id=cid,
text=text,
final_score=float(c.final_score),
fallback_level_final=int(selected_level_by_id.get(cid, last_fallback_level)),
explanations=explanations,
)
)
served_k = len(items)
# meta使用聚合统计确保单调性约束成立raw>=after_hard>=after_dedup>=after_freqcap>=served_k
# 注意:聚合统计理论上可能出现 after_* > raw_total例如 repo 返回重复/异常),此处交由 builder 防御修正
meta_builder.set_candidate_pool_size_raw(int(raw_total))
meta_builder.set_after_hard_filter(int(after_hard_total), risk_filtered_count_by_flag=dict(risk_counts_total))
meta_builder.set_after_dedup(int(after_dedup_total))
meta_builder.set_after_freqcap(int(after_freqcap_total), freqcap_filtered_counts=dict(freqcap_counts_total))
meta_builder.set_served_k(int(served_k))
meta_builder.set_fallback_level_final(int(last_fallback_level), reason=last_reason)
meta_builder.set_config_snapshot(
{
"fallback_trace": fallback_trace,
"engine_config": cfg.model_dump(),
"constraints": cons.model_dump(),
"locale": effective_locale,
}
)
return RecoEngineResult(items=items, meta=meta_builder.build())

View File

@@ -0,0 +1,101 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal, Optional
from pydantic import BaseModel, Field
from app.features.personalized_reco.observability.types import RecoMeta
Scene = Literal["feed", "push", "widget"]
class RecoConstraints(BaseModel):
"""
推荐请求的可选约束(调用方可按需传入)。
"""
exclude_content_ids: list[int] = Field(default_factory=list)
exclude_author_ids: list[str] = Field(default_factory=list)
exclude_template_ids: list[str] = Field(default_factory=list)
# 候选池上限(用于资源保护)
max_candidates_limit: Optional[int] = None
# Push/Widget 作者/模板冷却窗口内的历史集合(增强频控输入)
# 说明若不提供Nonererank_freqcap 会记录缺失并跳过该维度过滤
recent_author_ids: Optional[list[str]] = None
recent_template_ids: Optional[list[str]] = None
class RecoEngineConfig(BaseModel):
"""
引擎级配置V1 可调参项)。
"""
# Feed 是否允许 served_k < k允许不足
feed_allow_partial: bool = True
# Feed 是否在不足时继续回退补齐
feed_fill_with_fallback: bool = True
# 候选拉取倍率limit = min(max_candidates_limit, k * multiplier)
candidate_multiplier_feed: int = 10
candidate_multiplier_push_widget: int = 30
# 每层回退的最大候选数量下限(避免 k=1 但候选过少)
min_candidates_per_level: int = 30
# explanations 默认开启(但应保持轻量)
enable_explanations: bool = True
class RecommendedItem(BaseModel):
"""
引擎最终下发的推荐项。
"""
content_id: int
text: str
final_score: float
fallback_level_final: int
# 解释信息:默认开启,但建议保持轻量(避免 payload 过大)
explanations: Optional[dict[str, Any]] = None
class RecoEngineResult(BaseModel):
"""
引擎输出容器items + meta。
"""
items: list[RecommendedItem] = Field(default_factory=list)
meta: RecoMeta
class HardFilterResult(BaseModel):
"""
Hard Filter 输出。
"""
kept_items: list[Any] = Field(default_factory=list)
removed_count: int = 0
risk_filtered_count_by_flag: dict[str, int] = Field(default_factory=dict)
# 每条内容的命中信息(仅用于 explanations默认可为空
hits_by_content_id: dict[int, list[str]] = Field(default_factory=dict)
class RecommendRequest(BaseModel):
"""
内部便捷结构(单测/集成时可用)。
"""
scene: Scene
user_profile: Any
already_recommended_ids: list[Any] = Field(default_factory=list)
touched_or_viewed_ids: list[Any] = Field(default_factory=list)
k: int = 1
now: datetime
locale: Optional[str] = None
constraints: Optional[RecoConstraints] = None
config: Optional[RecoEngineConfig] = None

View File

@@ -0,0 +1,90 @@
from __future__ import annotations
from typing import Any, Iterable, Optional
def normalize_int_id_list(mixed_ids: Iterable[Any]) -> list[int]:
"""
将混合类型的 id 列表归一化为 int 列表。
规则:
- int/可转 int 的 str -> int
- 其他None/空字符串/不可解析)忽略
"""
out: list[int] = []
for x in mixed_ids or []:
if x is None:
continue
if isinstance(x, bool):
# 避免 True/False 被当作 1/0
continue
try:
s = str(x).strip()
if s == "":
continue
out.append(int(s))
except Exception:
continue
return out
def merge_exclude_ids(
*,
already_recommended_ids: Iterable[Any],
touched_or_viewed_ids: Iterable[Any],
extra_exclude_content_ids: Optional[Iterable[int]] = None,
) -> list[int]:
"""
合并并去重排除 id保持首次出现顺序
"""
merged = list(normalize_int_id_list(list(already_recommended_ids or []) + list(touched_or_viewed_ids or [])))
if extra_exclude_content_ids:
merged += [int(x) for x in extra_exclude_content_ids if x is not None]
seen: set[int] = set()
out: list[int] = []
for cid in merged:
if cid in seen:
continue
seen.add(cid)
out.append(cid)
return out
def normalize_or_default_locale(locale: Optional[str]) -> str:
"""
locale 防御式归一化:
- 未传/空 -> 默认 "en"
- 其他 -> 原样返回,由下游 normalize_locale 做严格校验
"""
if locale is None:
return "en"
raw = str(locale).strip()
return raw or "en"
def clamp_personalization_power(power: Any, *, fallback_level: int) -> float:
"""
按回退层级对 personalization_power 做防御式约束。
- L0不改
- L1<= 0.5
- L2/L3= 0
"""
try:
p = float(power)
except Exception:
p = 0.0
if p != p:
p = 0.0
if int(fallback_level) >= 2:
return 0.0
if int(fallback_level) >= 1:
return min(p, 0.5)
return p

View File

@@ -0,0 +1,22 @@
"""
个性化推荐Rerank & Freqcap 子模块(重排 / 去重 / 频控)
说明V1
- 本模块在 Soft Scoring 后执行,消费候选的 `final_score`,输出可下发的排序结果。
- 仅做 Dedup / Freqcap / Feed MMR不做 Soft Scoring 与 Hard Filter。
"""
from .defaults import get_default_config
from .rerank import rerank_and_freqcap
from .types import RerankConfig, RerankMeta, RerankResult, ScoredCandidate, Scene
__all__ = [
"RerankConfig",
"RerankMeta",
"RerankResult",
"ScoredCandidate",
"Scene",
"get_default_config",
"rerank_and_freqcap",
]

View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from app.features.personalized_reco.rerank_freqcap.types import RerankConfig, Scene
_DEFAULTS: dict[Scene, RerankConfig] = {
# FeedMMR λ=0.7;冷却参数不强制使用
"feed": RerankConfig(
mmr_lambda=0.7,
top_n_for_mmr=200,
cooldown_sentence_days=0,
cooldown_author_days=0,
cooldown_template_days=0,
),
# Push工程默认来自算法规则的建议参数
"push": RerankConfig(
mmr_lambda=0.7,
top_n_for_mmr=200,
cooldown_sentence_days=14,
cooldown_author_days=7,
cooldown_template_days=7,
),
# Widget工程默认
"widget": RerankConfig(
mmr_lambda=0.7,
top_n_for_mmr=200,
cooldown_sentence_days=7,
cooldown_author_days=7,
cooldown_template_days=7,
),
}
def get_default_config(scene: Scene) -> RerankConfig:
"""
获取指定场景的默认参数(返回副本,避免被意外修改)。
"""
base = _DEFAULTS[scene]
return RerankConfig.model_validate(base.model_dump())

View File

@@ -0,0 +1,208 @@
from __future__ import annotations
from typing import Any, Iterable, Optional
from app.features.personalized_reco.rerank_freqcap.defaults import get_default_config
from app.features.personalized_reco.rerank_freqcap.types import RerankConfig, RerankMeta, RerankResult, ScoredCandidate, Scene
from app.features.personalized_reco.rerank_freqcap.utils import as_finite_float, build_tags, clamp, jaccard, normalize_int_id_set
def _sort_by_score_desc(cands: list[ScoredCandidate]) -> list[ScoredCandidate]:
return sorted(cands, key=lambda x: as_finite_float(x.final_score, default=float("-inf")), reverse=True)
def _dedup_by_seen_ids(
cands: list[ScoredCandidate],
*,
seen_ids: set[int],
) -> tuple[list[ScoredCandidate], int]:
kept: list[ScoredCandidate] = []
removed = 0
for c in cands:
if int(c.content_id) in seen_ids:
removed += 1
continue
kept.append(c)
return kept, removed
def _apply_author_template_freqcap(
cands: list[ScoredCandidate],
*,
recent_author_ids: Optional[Iterable[str]],
recent_template_ids: Optional[Iterable[str]],
) -> tuple[list[ScoredCandidate], dict[str, int], list[str]]:
"""
V1 策略:
- 若 recent_*_ids 未提供None不执行该维度过滤但在 meta 记录缺失
- 若提供,则执行硬过滤
"""
filtered_counts: dict[str, int] = {"author": 0, "template": 0}
missing: list[str] = []
author_set: set[str] | None
if recent_author_ids is None:
author_set = None
missing.append("author")
else:
author_set = set([a for a in recent_author_ids if a is not None and str(a).strip() != ""])
template_set: set[str] | None
if recent_template_ids is None:
template_set = None
missing.append("template")
else:
template_set = set([t for t in recent_template_ids if t is not None and str(t).strip() != ""])
out: list[ScoredCandidate] = []
for c in cands:
if author_set is not None and c.author_id and c.author_id in author_set:
filtered_counts["author"] += 1
continue
if template_set is not None and c.template_id and c.template_id in template_set:
filtered_counts["template"] += 1
continue
out.append(c)
# 只返回真正生效的维度计数(避免 meta 噪音)
effective_counts: dict[str, int] = {}
if author_set is not None:
effective_counts["author"] = int(filtered_counts["author"])
if template_set is not None:
effective_counts["template"] = int(filtered_counts["template"])
missing_sorted = sorted(set(missing))
return out, effective_counts, missing_sorted
def _sim(a: ScoredCandidate, b: ScoredCandidate, *, tags_a: set[str], tags_b: set[str]) -> float:
# 离散特征版V1 推荐),对齐 plan.md
if int(a.content_id) == int(b.content_id):
return 1.0
sim = 0.0
if a.template_id and b.template_id and a.template_id == b.template_id:
sim += 0.6
if a.author_id and b.author_id and a.author_id == b.author_id:
sim += 0.3
sim += 0.1 * jaccard(tags_a, tags_b)
return clamp(sim, 0.0, 1.0)
def _mmr_rerank(
*,
candidates: list[ScoredCandidate],
k: int,
lam: float,
) -> list[ScoredCandidate]:
if k <= 0:
return []
if not candidates:
return []
lam_f = clamp(as_finite_float(lam, default=0.7), 0.0, 1.0)
# 预计算 tags避免重复构造
tags_map: dict[int, set[str]] = {}
for c in candidates:
tags_map[int(c.content_id)] = build_tags(c)
remaining = _sort_by_score_desc(list(candidates))
selected: list[ScoredCandidate] = []
# Top1最高分
selected.append(remaining.pop(0))
while remaining and len(selected) < k:
best_idx = 0
best_val = float("-inf")
for idx, c in enumerate(remaining):
rel = as_finite_float(c.final_score, default=float("-inf"))
tags_c = tags_map.get(int(c.content_id), set())
max_sim = 0.0
for s in selected:
tags_s = tags_map.get(int(s.content_id), set())
max_sim = max(max_sim, _sim(c, s, tags_a=tags_c, tags_b=tags_s))
val = lam_f * float(rel) - (1.0 - lam_f) * float(max_sim)
if val > best_val:
best_val = val
best_idx = idx
selected.append(remaining.pop(best_idx))
return selected
def rerank_and_freqcap(
*,
scene: Scene,
scored_candidates: list[ScoredCandidate],
already_recommended_ids: list[Any],
touched_or_viewed_ids: list[Any],
k: int,
config: Optional[RerankConfig] = None,
recent_author_ids: Optional[list[str]] = None,
recent_template_ids: Optional[list[str]] = None,
) -> RerankResult:
"""
主入口:对 scored_candidates 做去重/频控/重排,输出最终可下发序列。
V1 约定:
- 冷却窗口“按天”由调用方保证输入集合已经裁剪到窗口内,本模块以“集合代表窗口内历史”为准
- Feed 默认只做 dedup + MMRPush/Widget 做 dedup + freqcap + TopK
"""
cfg = config or get_default_config(scene)
# seen_ids = already_recommended_ids touched_or_viewed_ids
seen_ids = normalize_int_id_set(list(already_recommended_ids) + list(touched_or_viewed_ids))
# 先按分数降序,保证 Top1 与 TopK 一致
base_sorted = _sort_by_score_desc(list(scored_candidates))
after_dedup, removed_sentence = _dedup_by_seen_ids(base_sorted, seen_ids=seen_ids)
candidate_pool_size_after_dedup = len(after_dedup)
missing_history_fields: list[str] = []
freqcap_counts: dict[str, int] = {"sentence": int(removed_sentence)}
after_freqcap = after_dedup
# Push/Widget作者/模板冷却(增强项)
if scene in {"push", "widget"}:
after_freqcap, dim_counts, missing = _apply_author_template_freqcap(
after_freqcap,
recent_author_ids=recent_author_ids,
recent_template_ids=recent_template_ids,
)
missing_history_fields = missing
freqcap_counts.update(dim_counts)
else:
# Feed不强制作者/模板冷却V1 可选,这里默认跳过)
missing_history_fields = []
candidate_pool_size_after_freqcap = len(after_freqcap)
ranked: list[ScoredCandidate]
if scene == "feed":
# MMR 前截断,避免性能问题
top_n = int(cfg.top_n_for_mmr) if int(cfg.top_n_for_mmr) > 0 else len(after_freqcap)
mmr_pool = after_freqcap[:top_n]
ranked = _mmr_rerank(candidates=mmr_pool, k=int(k), lam=cfg.mmr_lambda)
else:
ranked = after_freqcap[: max(0, int(k))]
meta = RerankMeta(
candidate_pool_size_after_dedup=int(candidate_pool_size_after_dedup),
candidate_pool_size_after_freqcap=int(candidate_pool_size_after_freqcap),
missing_history_fields=missing_history_fields,
freqcap_filtered_counts=freqcap_counts,
)
return RerankResult(ranked_items=ranked, meta=meta)

View File

@@ -0,0 +1,61 @@
from __future__ import annotations
from typing import Any, Literal, Optional
from pydantic import BaseModel, Field
from app.features.personalized_reco.content_repository.types import ContentProfileDTO
Scene = Literal["feed", "push", "widget"]
class ScoredCandidate(BaseModel):
"""
Soft Scoring 后的候选项(本模块消费的最小字段集合)。
说明:
- `content_profile` 用于 Feed 的标签/相似度计算;缺失时需降级为仅使用 author/template 等字段
"""
content_id: int
final_score: float
author_id: Optional[str] = None
template_id: Optional[str] = None
content_profile: Optional[ContentProfileDTO] = None
# 允许透传额外字段(例如 text、breakdown 等),便于上层直接下发
extra: dict[str, Any] = Field(default_factory=dict)
class RerankConfig(BaseModel):
"""
重排/频控配置(可调参)。
"""
# FeedMMR
mmr_lambda: float = 0.7
top_n_for_mmr: int = 200
# Push/Widget冷却窗口V1 主要用于配置与可观测;真正按天需要带时间戳的历史)
cooldown_sentence_days: int = 14
cooldown_author_days: int = 7
cooldown_template_days: int = 7
class RerankMeta(BaseModel):
candidate_pool_size_after_dedup: int
candidate_pool_size_after_freqcap: int
# 例如未提供 recent_author_ids/recent_template_ids 时记录 ["author","template"]
missing_history_fields: list[str] = Field(default_factory=list)
# 可选但建议:按维度统计被过滤数量
freqcap_filtered_counts: dict[str, int] = Field(default_factory=dict)
class RerankResult(BaseModel):
ranked_items: list[ScoredCandidate] = Field(default_factory=list)
meta: RerankMeta

View File

@@ -0,0 +1,107 @@
from __future__ import annotations
import logging
from typing import Any, Iterable
logger = logging.getLogger(__name__)
def clamp(value: float, min_value: float, max_value: float) -> float:
if value != value: # NaN
return min_value
return max(min_value, min(max_value, value))
def as_finite_float(value: Any, *, default: float) -> float:
try:
f = float(value)
except Exception:
return float(default)
if f != f:
return float(default)
if f == float("inf") or f == float("-inf"):
return float(default)
return f
def normalize_int_id_set(values: Iterable[Any]) -> set[int]:
"""
将历史 ID 列表归一化为 int 集合(支持 str/int 混用)。
说明:
- 无法转换的值会被忽略,并记录 debug 日志(不影响主流程)
"""
out: set[int] = set()
for v in values:
try:
if isinstance(v, bool):
# 避免 True/False 被当作 1/0
raise ValueError("bool 不是合法 id")
out.add(int(v))
except Exception:
logger.debug("历史 id 无法转为 int已忽略%r", v)
return out
def jaccard(a: set[str], b: set[str]) -> float:
if not a and not b:
return 0.0
inter = len(a & b)
union = len(a | b)
return float(inter) / float(union) if union > 0 else 0.0
def argmax_key(d: dict[str, Any] | None) -> str | None:
"""
从 suitability 字典中取最大值 keyV1 用作代表标签)。
- 空字典/None -> None
- 值非法 -> 按 default=0 处理
"""
if not d:
return None
best_k: str | None = None
best_v = float("-inf")
for k, v in d.items():
fv = as_finite_float(v, default=0.0)
if fv > best_v:
best_v = fv
best_k = k
return best_k
def build_tags(candidate: Any) -> set[str]:
"""
构造离散标签集合V1 写死):
- stage:<stage>
- need:<argmax_key>
- context:<argmax_key>
说明:
- candidate 可能是 ScoredCandidate 或具备 content_profile 的对象
- 字段缺失时自动降级(只返回可得标签)
"""
tags: set[str] = set()
cp = getattr(candidate, "content_profile", None)
if cp is None:
return tags
stage = getattr(cp, "stage", None)
if stage:
tags.add(f"stage:{stage}")
need = getattr(cp, "need_suitability", None)
need_k = argmax_key(need)
if need_k:
tags.add(f"need:{need_k}")
ctx = getattr(cp, "context_suitability", None)
ctx_k = argmax_key(ctx)
if ctx_k:
tags.add(f"context:{ctx_k}")
return tags

View File

@@ -0,0 +1,22 @@
"""
个性化推荐Scoring 子模块(软打分与惩罚项)
说明:
- 本模块只做软打分与本模块定义的惩罚项P_uncertainty、Widget 情绪软降权)。
- Hard Filter / 频控重排 / 新鲜度等由其他模块产出,通过入参注入(缺省按 0
"""
from .defaults import get_default_config
from .score import score_content
from .types import ExternalTerms, Scene, ScoreBreakdown, ScoreConfig, ScoreResult
__all__ = [
"ExternalTerms",
"Scene",
"ScoreBreakdown",
"ScoreConfig",
"ScoreResult",
"get_default_config",
"score_content",
]

View File

@@ -0,0 +1,44 @@
from __future__ import annotations
from app.features.personalized_reco.scoring.types import Scene, ScoreConfig
_DEFAULTS: dict[Scene, ScoreConfig] = {
# 来源:设计说明文档/個性化推薦算法規則.mdV1 建议权重)
"feed": ScoreConfig(
w_need=0.35,
w_emotion=0.20,
w_stage=0.15,
w_context=0.30,
# Feed 默认不启用不确定性惩罚(可按需开启)
enable_uncertainty_penalty=False,
),
"push": ScoreConfig(
w_need=0.45,
w_emotion=0.35,
w_stage=0.15,
w_context=0.05,
# Push 默认启用不确定性惩罚
enable_uncertainty_penalty=True,
),
"widget": ScoreConfig(
w_need=0.25,
w_emotion=0.25,
w_stage=0.30,
w_context=0.20,
# Widget 默认不启用不确定性惩罚(可按需开启)
enable_uncertainty_penalty=False,
widget_emotion_soft_range=(0.4, 0.8),
widget_emotion_penalty_gamma=0.25,
),
}
def get_default_config(scene: Scene) -> ScoreConfig:
"""
获取指定场景的默认打分参数(返回副本,避免被意外修改)。
"""
base = _DEFAULTS[scene]
return ScoreConfig.model_validate(base.model_dump())

View File

@@ -0,0 +1,201 @@
from __future__ import annotations
from datetime import datetime
from typing import Optional
from app.features.personalized_reco.content_repository.types import ContentProfileDTO
from app.features.personalized_reco.scoring.defaults import get_default_config
from app.features.personalized_reco.scoring.types import ExternalTerms, Scene, ScoreBreakdown, ScoreConfig, ScoreResult
from app.features.personalized_reco.scoring.utils import as_finite_float, clamp, pick_one_hot_key
from app.features.user_profile_scoring.types import UserProfileV1_2
def _missing_fields(user_profile: UserProfileV1_2) -> list[str]:
missing: list[str] = []
if not user_profile.need:
missing.append("need")
if not user_profile.context:
missing.append("context")
if user_profile.emotion_score is None:
missing.append("emotion")
return missing
def _score_need(user_profile: UserProfileV1_2, content: ContentProfileDTO) -> float:
key = pick_one_hot_key(user_profile.need) # type: ignore[arg-type]
if key is None:
return 0.5
raw = content.need_suitability.get(key, 0.5)
return clamp(as_finite_float(raw, default=0.5), 0.0, 1.0)
def _score_context(user_profile: UserProfileV1_2, content: ContentProfileDTO) -> float:
key = pick_one_hot_key(user_profile.context) # type: ignore[arg-type]
if key is None:
return 0.5
raw = content.context_suitability.get(key, 0.5)
return clamp(as_finite_float(raw, default=0.5), 0.0, 1.0)
def _score_emotion(user_profile: UserProfileV1_2, content: ContentProfileDTO) -> float:
# V1.2:用户情绪缺失 -> 0.8
if user_profile.emotion_score is None:
return 0.8
# 文案 generalemotion_score=None-> 0.8
if content.emotion_score is None:
return 0.8
u = clamp(as_finite_float(user_profile.emotion_score, default=0.8), 0.0, 1.0)
c = clamp(as_finite_float(content.emotion_score, default=0.8), 0.0, 1.0)
return clamp(1.0 - abs(u - c), 0.0, 1.0)
def _user_stage_key(user_profile: UserProfileV1_2) -> str:
# 约定UserStageOneHot.unknown 必填;但这里仍做防御
stage = user_profile.stage
if getattr(stage, "expecting", 0) == 1:
return "expecting"
if getattr(stage, "parenting", 0) == 1:
return "parenting"
if getattr(stage, "unknown", 1) == 1:
return "unknown"
return "unknown"
def _score_stage(user_profile: UserProfileV1_2, content: ContentProfileDTO) -> float:
# 对齐算法规则:
# - general=1命中=1unknown对非unknown=0.7;其余=0
if content.stage == "general":
return 1.0
u_stage = _user_stage_key(user_profile)
if content.stage == u_stage:
return 1.0
if u_stage == "unknown" and content.stage != "unknown":
return 0.7
return 0.0
def _score_personal(alpha: float, personalization_power: float, s_need: float, s_context: float) -> float:
power = clamp(as_finite_float(personalization_power, default=0.0), 0.0, 1.0)
a = as_finite_float(alpha, default=0.0)
return float(a) * float(power) * max(float(s_need), float(s_context))
def _penalty_uncertainty(beta: float, user_profile: UserProfileV1_2, content: ContentProfileDTO) -> float:
b = as_finite_float(beta, default=0.0)
power = clamp(as_finite_float(content.personalization_power, default=0.0), 0.0, 1.0)
# V1 约定conf_U 缺失时按 1.0(避免过惩罚)
conf_u = clamp(as_finite_float(getattr(user_profile, "profile_confidence", 1.0), default=1.0), 0.0, 1.0)
conf_c = clamp(as_finite_float(getattr(content, "review_confidence", 0.7), default=0.7), 0.0, 1.0)
return float(b) * (1.0 - float(conf_u)) * (1.0 - float(conf_c)) * float(power)
def _widget_emotion_penalty(scene: Scene, content: ContentProfileDTO, config: ScoreConfig) -> float:
if scene != "widget":
return 0.0
if content.emotion_score is None:
return 0.0
lo, hi = config.widget_emotion_soft_range
lo_f = as_finite_float(lo, default=0.4)
hi_f = as_finite_float(hi, default=0.8)
width = hi_f - lo_f
if width <= 0:
return 0.0
e = clamp(as_finite_float(content.emotion_score, default=0.6), 0.0, 1.0)
if e < lo_f:
d = lo_f - e
elif e > hi_f:
d = e - hi_f
else:
d = 0.0
gamma = as_finite_float(config.widget_emotion_penalty_gamma, default=0.25)
raw = float(gamma) * float(d) / float(width)
return clamp(raw, 0.0, float(gamma))
def score_content(
*,
scene: Scene,
user_profile: UserProfileV1_2,
content_profile: ContentProfileDTO,
config: Optional[ScoreConfig] = None,
pass_filters: bool = True,
external_terms: Optional[ExternalTerms] = None,
now: Optional[datetime] = None, # 预留V1 不使用
) -> ScoreResult:
"""
主入口:对单条内容 Cᵢ 进行软打分,返回 final_score 与 breakdown。
说明V1
- `pass_filters` 来自 Hard Filter本模块不做硬过滤
- `external_terms` 可注入 S_fresh / P_fatigue / P_repeat / P_risk缺省按 0
- `now` 预留给未来的 freshness/时间衰减V1 不实现)
"""
cfg = config or get_default_config(scene)
ext = external_terms or ExternalTerms()
missing = _missing_fields(user_profile)
s_need = _score_need(user_profile, content_profile)
s_context = _score_context(user_profile, content_profile)
s_emotion = _score_emotion(user_profile, content_profile)
s_stage = _score_stage(user_profile, content_profile)
w_need = as_finite_float(cfg.w_need, default=0.0)
w_emotion = as_finite_float(cfg.w_emotion, default=0.0)
w_stage = as_finite_float(cfg.w_stage, default=0.0)
w_context = as_finite_float(cfg.w_context, default=0.0)
s_core = float(w_need) * s_need + float(w_emotion) * s_emotion + float(w_stage) * s_stage + float(w_context) * s_context
s_personal = _score_personal(cfg.alpha, content_profile.personalization_power, s_need, s_context)
p_uncertainty = 0.0
if cfg.enable_uncertainty_penalty:
p_uncertainty = _penalty_uncertainty(cfg.beta, user_profile, content_profile)
p_widget = _widget_emotion_penalty(scene, content_profile, cfg)
s_fresh = as_finite_float(ext.S_fresh, default=0.0)
p_fatigue = as_finite_float(ext.P_fatigue, default=0.0)
p_repeat = as_finite_float(ext.P_repeat, default=0.0)
p_risk_external = as_finite_float(ext.P_risk, default=0.0)
# Widget 软降权并入 P_risk但在 breakdown 中单独暴露,便于打点)
p_risk = float(p_risk_external) + float(p_widget)
raw_final = s_core + s_personal + float(s_fresh) - float(p_fatigue) - float(p_repeat) - float(p_risk) - float(p_uncertainty)
final_score = float(raw_final) if pass_filters else 0.0
breakdown = ScoreBreakdown(
scene=scene,
**{
"pass": bool(pass_filters),
},
missing_fields=missing,
S_need=float(s_need),
S_context=float(s_context),
S_stage=float(s_stage),
S_emotion=float(s_emotion),
S_core=float(s_core),
S_personal=float(s_personal),
S_fresh=float(s_fresh),
P_fatigue=float(p_fatigue),
P_repeat=float(p_repeat),
P_risk=float(p_risk),
P_uncertainty=float(p_uncertainty),
P_widget_emotion_out_of_range=float(p_widget),
)
return ScoreResult(final_score=float(final_score), breakdown=breakdown)

View File

@@ -0,0 +1,84 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field
Scene = Literal["feed", "push", "widget"]
class ScoreConfig(BaseModel):
"""
打分配置(可调参)。
说明:
- 默认值由 `defaults.get_default_config(scene)` 提供
- 本模块不负责回退梯度fallback_level策略仅做防御式 clamp
"""
w_need: float
w_emotion: float
w_stage: float
w_context: float
alpha: float = 0.15
beta: float = 0.30
enable_uncertainty_penalty: bool = False
# Widget 情绪软区间与软降权强度
widget_emotion_soft_range: tuple[float, float] = (0.4, 0.8)
widget_emotion_penalty_gamma: float = 0.25
class ExternalTerms(BaseModel):
"""
外部注入项V1 可选)。
说明:
- 由 `rerank-freqcap` 或 `reco-engine` 产出
- 本模块缺省按 0保证可排序与输出结构稳定
"""
S_fresh: float = 0.0
P_fatigue: float = 0.0
P_repeat: float = 0.0
P_risk: float = 0.0
class ScoreBreakdown(BaseModel):
"""
可观测分解项(用于调参与回归测试)。
"""
scene: Scene
passed: bool = Field(alias="pass")
missing_fields: list[str] = Field(default_factory=list)
S_need: float
S_context: float
S_stage: float
S_emotion: float
S_core: float
S_personal: float
S_fresh: float
P_fatigue: float
P_repeat: float
P_risk: float
P_uncertainty: float
# Widget 专用:区间外软降权(建议保留,便于打点)
P_widget_emotion_out_of_range: float = 0.0
model_config = {
"populate_by_name": True,
}
class ScoreResult(BaseModel):
final_score: float
breakdown: ScoreBreakdown

View File

@@ -0,0 +1,59 @@
from __future__ import annotations
import logging
from typing import Any
logger = logging.getLogger(__name__)
def clamp(value: float, min_value: float, max_value: float) -> float:
"""
将值裁剪到区间内,并对 NaN 做兜底。
"""
if value != value: # NaN
return min_value
return max(min_value, min(max_value, value))
def as_finite_float(value: Any, *, default: float) -> float:
"""
将任意值尽量转为有限 float失败则返回 default。
"""
try:
f = float(value)
except Exception:
return float(default)
# NaN / inf 都视为不可用
if f != f:
return float(default)
if f == float("inf") or f == float("-inf"):
return float(default)
return f
def pick_one_hot_key(one_hot: dict[str, Any] | None) -> str | None:
"""
从稀疏 one-hot{key: 1})中取唯一 key。
约定:
- None / {} → 缺失,返回 None
- 单 key → 返回该 key
- 多 key → 取“字典序最小”的 key并记录 debug 日志(避免静默歧义)
"""
if not one_hot:
return None
keys = [k for k, v in one_hot.items() if v == 1 or v is True]
if not keys:
return None
if len(keys) == 1:
return keys[0]
chosen = sorted(keys)[0]
logger.debug("one-hot 出现多个 key=1已按字典序选择chosen=%s keys=%s", chosen, keys)
return chosen