Add keyword cleanup governance workflow

src/summary_mcp/workflows/freshrss_pipeline.py

@@ -6,6 +6,7 @@ from datetime import UTC, date, datetime
 from pathlib import Path
 from typing import Any
 
+from summary_mcp.core.keyword_index import persist_keyword_indexes
 from summary_mcp.core.pipeline import extract_content
 from summary_mcp.core.summary_loop import resolve_llm_settings, run_loop_payload
 from summary_mcp.filters.engine import evaluate_filter_rules, load_filter_rules
@@ -26,8 +27,13 @@ from summary_mcp.models.summary_io import ExtractionInput
 REPO_ROOT = Path(__file__).resolve().parents[3]
 OUTPUT_ROOT = REPO_ROOT / "outputs"
 FRESHRSS_OUTPUT_ROOT = OUTPUT_ROOT / "freshrss"
+DATA_ROOT = REPO_ROOT / "data" / "term_index"
 DEFAULT_PROMPT_PATH = OUTPUT_ROOT / "prompts" / "llm-summary-prompt.txt"
 DEFAULT_RULES_PATH = REPO_ROOT / "configs" / "filter_rules.json"
+DEFAULT_TERM_ALIASES_PATH = REPO_ROOT / "configs" / "term_aliases.json"
+DEFAULT_TERM_STOPWORDS_PATH = REPO_ROOT / "configs" / "term_stopwords.json"
+DEFAULT_TERM_DAILY_DIR = DATA_ROOT / "daily"
+DEFAULT_TERM_STATS_PATH = DATA_ROOT / "term_stats.json"
 
 
 def _save_json(path: Path, payload: dict[str, Any] | list[Any]) -> None:
@@ -237,6 +243,17 @@ def run_freshrss_pipeline(
     )
     _save_json(delivery_output, delivery_payload.model_dump(mode="json"))
 
+    keyword_index_result = persist_keyword_indexes(
+        delivery_payload.candidates,
+        for_date=delivery_payload.date,
+        digest_id=delivery_payload.run_id,
+        source="openclaw_delivery_payload",
+        daily_dir=DEFAULT_TERM_DAILY_DIR,
+        stats_path=DEFAULT_TERM_STATS_PATH,
+        aliases_path=DEFAULT_TERM_ALIASES_PATH,
+        stopwords_path=DEFAULT_TERM_STOPWORDS_PATH,
+    )
+
     marked_count = 0
     if mark_read and delivered_item_ids:
         client.mark_items_as_read(auth_token=auth_token, item_ids=delivered_item_ids)
@@ -259,6 +276,7 @@ def run_freshrss_pipeline(
         "debug_artifacts": debug_artifacts,
         "raw_output": str(raw_output),
         "delivery_output": str(delivery_output),
+        "keyword_index": keyword_index_result,
         "status_counts": status_counts,
         "items": item_reports,
     }
@@ -270,6 +288,7 @@ def run_freshrss_pipeline(
         "raw_output": str(raw_output),
         "delivery_output": str(delivery_output),
         "report_output": str(report_output),
+        "keyword_index": keyword_index_result,
         "pulled_count": len(items),
         "delivered_count": len(delivered_candidates),
         "marked_read_count": marked_count,