From b5ec58cd6aa171d8aebfdb07da37d34942f11e7c Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Apr 2026 16:48:03 +0800 Subject: [PATCH] feat: add public digest brief output for openclaw --- README.md | 1 + docs/openclaw/digest-optimization-summary.md | 298 ++++++++++++++++++ src/summary_mcp/models/__init__.py | 4 + src/summary_mcp/models/openclaw_delivery.py | 58 +++- .../workflows/freshrss_pipeline.py | 22 +- 5 files changed, 377 insertions(+), 6 deletions(-) create mode 100644 docs/openclaw/digest-optimization-summary.md diff --git a/README.md b/README.md index a825dde..7a25030 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ python scripts/run_freshrss_pipeline.py ^ - `outputs/freshrss/rerun//raw/freshrss.raw.json` - `outputs/freshrss/rerun//candidates/openclaw-delivery-payload.json` +- `outputs/freshrss/rerun//candidates/digest-brief.json`（给 OpenClaw 生成 public digest 用的轻量输入，仅包含 `keep` 候选） - `outputs/freshrss/rerun//run-report.json` - `outputs/freshrss/rerun//extracted/item-XX.extracted.json`（每篇一份） diff --git a/docs/openclaw/digest-optimization-summary.md b/docs/openclaw/digest-optimization-summary.md new file mode 100644 index 0000000..ad22ab5 --- /dev/null +++ b/docs/openclaw/digest-optimization-summary.md @@ -0,0 +1,298 @@ +# Digest Optimization Summary + +## 背景 + +reader → OpenClaw 日报链路原先的问题主要有两类： + +1. **OpenClaw public digest 输入过重** + - public digest 直接读取完整 `openclaw-delivery-payload.json` + - 其中混有大量不直接服务公开日报的字段 + - public digest 这一步在 OpenClaw 侧消耗了较多 token + +2. **public / internal 生成逻辑没有充分拆分** + - public digest 与 internal review digest 都基于完整 payload 推导 + - 容易造成重复消耗 + - public digest 还可能被 review / 内部流程语义污染 + +本轮优化的目标不是重写 reader 主流程，而是在不破坏现有 delivery payload 的前提下，先把 public digest 的输入和生成方式收敛下来，并验证整体 token 与内容质量的变化。 + +--- + +## 本轮改动 + +### 1. reader 新增 `digest-brief.json` + +在 FreshRSS pipeline 写出： + +- `outputs/freshrss/rerun//candidates/openclaw-delivery-payload.json` + +之后，额外生成： + +- `outputs/freshrss/rerun//candidates/digest-brief.json` + +用途： + +- 供 OpenClaw 生成 **public digest** 时优先读取 +- 作为 public-only 的轻量输入视图 + +当前约束： + +- 仅保留 `selection_decision == "keep"` 的候选 +- 默认最多保留前 5 条 +- 高亮 `highlights` 最多保留 3 条 +- schema 标识为 `digest-brief.v1` + +保留字段： + +- `title` +- `source_name` +- `summary` +- `highlights` +- `category` +- `digest_rank` +- `selection_decision` +- `url` + +附带计数： + +- `source_candidate_count` +- `candidate_count` + +--- + +### 2. public / internal 输入边界拆分 + +当前推荐口径： + +- **public digest** + - 优先读取 `digest-brief.json` + - 仅使用 public-only 输入视图 + +- **internal review digest** + - 继续读取完整 `openclaw-delivery-payload.json` + - 保留 keep / review 的决策上下文 + +这样做的原因： + +- public digest 需要更轻、更干净的公开输入 +- internal review digest 仍然需要完整上下文来支撑判断、待确认与建议沉淀 + +--- + +### 3. 一次生成两份 digest 的生成模式 + +推荐把： + +- `public digest` +- `internal review digest` + +改为在 OpenClaw 侧 **一次调用同时生成两份**。 + +推荐输入： + +- `PUBLIC_DIGEST_INPUT` → `digest-brief.json` +- `INTERNAL_REVIEW_INPUT` → `openclaw-delivery-payload.json` + +推荐输出： + +```json +{ + "public_digest_markdown": "...", + "internal_review_digest_markdown": "..." +} +``` + +这样可以减少重复 prompt / 调用开销，同时保留 public / internal 两种视图的边界。 + +--- + +### 4. internal review digest 风格收敛 + +internal review digest 经过一轮人工验证后，收敛成以下规则： + +固定结构： + +1. `今日候选概况` +2. `已入选重点` +3. `待你确认` +4. `建议沉淀到 IMA` +5. `原始候选清单` + +表达规则： + +- 不显示 `rank` +- 不显示英文 machine state +- 使用中文状态： + - `keep` → `已入选` + - `review` → `待确认` + - `drop` → `暂不纳入` + +内容规则： + +- `已入选重点` + - 标题 + 来源 + - 状态 + - 较完整的一段摘要 + - 一段判断（解释为什么值得入选，以及它在今天 digest 中承担什么角色） + +- `待你确认` + - 标题 + 来源 + - 状态 + - 较完整的一段摘要 + - 原因 + - 建议 + +- `原始候选清单` + - 也用中文状态，而不是 `decision=keep/review` + +目标： + +- 保留 internal review digest 作为“给人看的审阅稿”的属性 +- 避免它沦为 payload 的原样转写或机器中间态展示 + +--- + +### 5. public digest 风格收敛 + +public digest 当前推荐结构： + +1. `今日概览` +2. `今日重点` +3. `趋势观察` +4. `延伸阅读` +5. `信息来源` + +表达规则： + +- 不暴露 internal workflow 词汇 +- 不写 `待确认` / `建议沉淀到 IMA` / `keep/review/drop` / `selection_decision` +- 保持适合 Hugo 公开浏览的表达方式 + +内容规则： + +- 每个 `今日重点` 条目除了摘要和 highlights 外，增加一句编辑性总结 +- 推荐形式： + - `这篇内容更值得关注的原因在于……` + +目标： + +- 保证 public digest 不只是“摘要列表” +- 而是一份带有编辑性提炼的公开日报 + +--- + +## 实测结果 + +基于真实 run： + +- run 目录：`outputs/freshrss/rerun/20260401-074614` + +### 1. public 输入压缩效果 + +- 完整 payload：`8701` 字符 +- `digest-brief.json`：`3080` 字符 + +压缩比例： + +- **减少约 64.6%** + +按中位 token 粗估： + +- 完整 payload：约 `3955 tokens` +- public brief：约 `1400 tokens` + +public 输入侧单次大约减少： + +- **约 2500 tokens** + +--- + +### 2. 一次生成两份的总成本估算 + +基于真实输入输出的中位估算： + +- 旧方案（两次生成）：约 `10328 tokens` +- 新方案（一次生成两份）：约 `7734 tokens` + +节省： + +- **约 2594 tokens** +- **约 25%** + +说明： + +- 第一步 public 输入瘦身带来的是“输入量级下降” +- 第二步一次生成两份带来的是“调用层重复开销下降” +- 两者叠加后，已经形成比较明显的成本优化效果 + +--- + +## 当前默认口径 + +### public digest + +- 输入：`digest-brief.json` +- 风格：公开浏览稿 +- 每个重点项包含： + - 摘要 + - 关键信号 + - 一句编辑性总结 + +### internal review digest + +- 输入：完整 payload +- 风格：内部审阅稿 +- 每个重点项包含： + - 更完整摘要 + - 判断 +- 每个待确认项包含： + - 更完整摘要 + - 原因 + - 建议 + +### 生成方式 + +- 优先采用 **一次调用同时生成两份** + +--- + +## 当前阶段结论 + +本轮优化已经形成一个可用版本： + +- reader 新增 public-only 轻量输入视图 +- public / internal 边界清楚 +- internal 风格和 public 风格都收敛到了可接受版本 +- token 成本下降有明确实测支撑 +- skill 文档与流程规范已经同步更新 + +当前更适合的策略不是继续抽象设计，而是： + +- 按这套新流程再跑几次真实日报 +- 观察稳定性、质量波动和实际使用感受 + +--- + +## 后续可选方向 + +### 1. internal 输入进一步轻量化 + +潜在方向： + +- 新增一个 internal 专用的轻量视图 +- 但需要谨慎，避免削弱 internal review 的判断价值 + +### 2. digest 阶段模型分层 + +潜在方向： + +- reader 上游继续用便宜模型做抽取和结构化 +- OpenClaw digest 阶段单独切到更便宜或更合适的模型 + +### 3. 自动化执行收口 + +潜在方向： + +- 把“一次生成两份”的逻辑进一步标准化 +- 更顺滑地接 Hugo 发布与聊天回传 +- 让 reader → OpenClaw → Hugo / chat 的路径更接近真正的稳定生产流程 diff --git a/src/summary_mcp/models/__init__.py b/src/summary_mcp/models/__init__.py index 2da5477..6236b84 100644 --- a/src/summary_mcp/models/__init__.py +++ b/src/summary_mcp/models/__init__.py @@ -26,7 +26,9 @@ from .keyword_index import DailyKeywordIndex, DailyKeywordTerm, KeywordStat, Key from .openclaw_delivery import ( OpenClawDeliveryPayload, OpenClawDeliveryStats, + OpenClawDigestBrief, build_openclaw_delivery_payload, + build_openclaw_digest_brief, ) __all__ = [ @@ -50,10 +52,12 @@ __all__ = [ "OpenClawCandidateInput", "OpenClawDeliveryPayload", "OpenClawDeliveryStats", + "OpenClawDigestBrief", "ReviewState", "ReviewStatus", "build_article_candidate_record", "build_openclaw_delivery_payload", + "build_openclaw_digest_brief", "build_openclaw_candidate_input", "candidate_id_for", "normalize_candidate_url", diff --git a/src/summary_mcp/models/openclaw_delivery.py b/src/summary_mcp/models/openclaw_delivery.py index a49df23..92b3ba8 100644 --- a/src/summary_mcp/models/openclaw_delivery.py +++ b/src/summary_mcp/models/openclaw_delivery.py @@ -6,6 +6,9 @@ from pydantic import BaseModel, Field from .article_candidate import OpenClawCandidateInput +DIGEST_BRIEF_LIMIT = 5 +DIGEST_BRIEF_HIGHLIGHT_LIMIT = 3 + class OpenClawDeliveryStats(BaseModel): total: int = Field(default=0, ge=0) @@ -23,6 +26,27 @@ class OpenClawDeliveryPayload(BaseModel): stats: OpenClawDeliveryStats = Field(default_factory=OpenClawDeliveryStats) +class DigestBriefCandidate(BaseModel): + title: str + source_name: str | None = None + summary: str + highlights: list[str] = Field(default_factory=list) + category: str | None = None + digest_rank: int = Field(default=0) + selection_decision: str + url: str + + +class OpenClawDigestBrief(BaseModel): + schema_version: str = "digest-brief.v1" + generated_at: datetime + run_id: str + date: date + source_candidate_count: int = Field(default=0, ge=0) + candidate_count: int = Field(default=0, ge=0) + top_candidates: list[DigestBriefCandidate] = Field(default_factory=list) + + def build_openclaw_delivery_payload( candidates: list[OpenClawCandidateInput], *, @@ -46,4 +70,36 @@ def build_openclaw_delivery_payload( date=payload_date, candidates=candidates, stats=stats, - ) \ No newline at end of file + ) + + +def build_openclaw_digest_brief( + payload: OpenClawDeliveryPayload, + *, + limit: int = DIGEST_BRIEF_LIMIT, + highlight_limit: int = DIGEST_BRIEF_HIGHLIGHT_LIMIT, +) -> OpenClawDigestBrief: + keep_candidates = [candidate for candidate in payload.candidates if candidate.selection_decision == "keep"] + sorted_candidates = sorted(keep_candidates, key=lambda candidate: candidate.digest_rank, reverse=True) + top_candidates = [ + DigestBriefCandidate( + title=candidate.title, + source_name=candidate.source_name, + summary=candidate.summary, + highlights=list(candidate.highlights[:highlight_limit]), + category=candidate.category, + digest_rank=candidate.digest_rank, + selection_decision=candidate.selection_decision, + url=str(candidate.url), + ) + for candidate in sorted_candidates[:limit] + ] + return OpenClawDigestBrief( + schema_version="digest-brief.v1", + generated_at=payload.generated_at, + run_id=payload.run_id, + date=payload.date, + source_candidate_count=len(payload.candidates), + candidate_count=len(top_candidates), + top_candidates=top_candidates, + ) diff --git a/src/summary_mcp/workflows/freshrss_pipeline.py b/src/summary_mcp/workflows/freshrss_pipeline.py index 48a14bc..1ec410d 100644 --- a/src/summary_mcp/workflows/freshrss_pipeline.py +++ b/src/summary_mcp/workflows/freshrss_pipeline.py @@ -26,7 +26,11 @@ from summary_mcp.models.article_candidate import ( ) from summary_mcp.models.filtering import FilterContext, FilterInput from summary_mcp.models.llm_result import LlmSummaryResult -from summary_mcp.models.openclaw_delivery import OpenClawDeliveryPayload, build_openclaw_delivery_payload +from summary_mcp.models.openclaw_delivery import ( + OpenClawDeliveryPayload, + build_openclaw_delivery_payload, + build_openclaw_digest_brief, +) from summary_mcp.models.summary_io import ExtractionInput @@ -197,8 +201,8 @@ def _build_and_persist_delivery( resolved_run_id: str, resolved_delivery_date: Any, delivery_output: Path, -) -> tuple[OpenClawDeliveryPayload, dict[str, Any]]: - # 按 digest_rank 排序，构建 delivery payload，持久化词元索引 +) -> tuple[OpenClawDeliveryPayload, Path, dict[str, Any]]: + # 按 digest_rank 排序，构建 delivery payload；同时派生一个更轻量的 digest-brief.json 供日报生成使用。 delivered_candidates.sort(key=lambda candidate: candidate.digest_rank, reverse=True) delivery_payload = build_openclaw_delivery_payload( delivered_candidates, @@ -207,6 +211,10 @@ def _build_and_persist_delivery( ) _save_json(delivery_output, delivery_payload.model_dump(mode="json")) + digest_brief = build_openclaw_digest_brief(delivery_payload) + digest_brief_output = delivery_output.with_name("digest-brief.json") + _save_json(digest_brief_output, digest_brief.model_dump(mode="json")) + keyword_index_result = persist_keyword_indexes( delivery_payload.candidates, for_date=delivery_payload.date, @@ -217,7 +225,7 @@ def _build_and_persist_delivery( aliases_path=DEFAULT_TERM_ALIASES_PATH, stopwords_path=DEFAULT_TERM_STOPWORDS_PATH, ) - return delivery_payload, keyword_index_result + return delivery_payload, digest_brief_output, keyword_index_result def _build_run_report( @@ -232,6 +240,7 @@ def _build_run_report( debug_artifacts: bool, raw_output: Path, delivery_output: Path, + digest_brief_output: Path, keyword_index_result: dict[str, Any], item_reports: list[dict[str, Any]], ) -> dict[str, Any]: @@ -253,6 +262,7 @@ def _build_run_report( "debug_artifacts": debug_artifacts, "raw_output": str(raw_output), "delivery_output": str(delivery_output), + "digest_brief_output": str(digest_brief_output), "keyword_index": keyword_index_result, "status_counts": status_counts, "items": item_reports, @@ -369,7 +379,7 @@ def run_freshrss_pipeline( item_reports.append(result) # --- 阶段 4+5：构建 delivery payload 并持久化词元索引 --- - delivery_payload, keyword_index_result = _build_and_persist_delivery( + delivery_payload, digest_brief_output, keyword_index_result = _build_and_persist_delivery( delivered_candidates=delivered_candidates, resolved_run_id=resolved_run_id, resolved_delivery_date=resolved_delivery_date, @@ -394,6 +404,7 @@ def run_freshrss_pipeline( debug_artifacts=debug_artifacts, raw_output=raw_output, delivery_output=delivery_output, + digest_brief_output=digest_brief_output, keyword_index_result=keyword_index_result, item_reports=item_reports, ) @@ -404,6 +415,7 @@ def run_freshrss_pipeline( "output_dir": str(resolved_output_dir), "raw_output": str(raw_output), "delivery_output": str(delivery_output), + "digest_brief_output": str(digest_brief_output), "report_output": str(report_output), "keyword_index": keyword_index_result, "pulled_count": len(items),