Add FreshRSS ingestion and rule filtering

2026-03-24 18:37:14 +08:00
parent c8d96705d1
commit ff3d15b8c4
21 changed files with 1460 additions and 21 deletions
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SRC_ROOT = REPO_ROOT / "src"
+
+if str(SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(SRC_ROOT))
+
+from summary_mcp.integrations.freshrss import FreshRSSClient, map_entry_to_item
+
+
+def _save_json(path: Path, payload: dict[str, Any] | list[Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _load_required(name: str, value: str | None) -> str:
+    resolved = value or os.environ.get(name)
+    if not resolved:
+        cli_name = name.lower().replace("_", "-")
+        raise RuntimeError(f"Missing required value: pass --{cli_name} or set {name}.")
+    return resolved
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Pull FreshRSS entries and map them into normalized item objects.")
+    parser.add_argument("--api-base-url", type=str, default=None, help="FreshRSS greader API base URL")
+    parser.add_argument("--username", type=str, default=None, help="FreshRSS API username")
+    parser.add_argument("--api-password", type=str, default=None, help="FreshRSS API password")
+    parser.add_argument(
+        "--stream-id",
+        type=str,
+        default="user/-/state/com.google/reading-list",
+        help="Google Reader API stream id",
+    )
+    parser.add_argument("--limit", type=int, default=10, help="Maximum number of entries to request")
+    parser.add_argument("--continuation", type=str, default=None, help="Continuation token for paging")
+    parser.add_argument(
+        "--raw-output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "freshrss.raw.json",
+        help="Where to save the raw FreshRSS response",
+    )
+    parser.add_argument(
+        "--items-output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "freshrss.items.json",
+        help="Where to save the mapped item list",
+    )
+    parser.add_argument("--timeout", type=float, default=20.0, help="Request timeout in seconds")
+    args = parser.parse_args()
+
+    api_base_url = _load_required("FRESHRSS_API_BASE_URL", args.api_base_url)
+    username = _load_required("FRESHRSS_USERNAME", args.username)
+    api_password = _load_required("FRESHRSS_API_PASSWORD", args.api_password)
+
+    client = FreshRSSClient(
+        api_base_url=api_base_url,
+        username=username,
+        api_password=api_password,
+        timeout_seconds=args.timeout,
+    )
+    auth_token = client.client_login()
+    payload = client.fetch_stream_contents(
+        auth_token=auth_token,
+        stream_id=args.stream_id,
+        limit=args.limit,
+        continuation=args.continuation,
+    )
+
+    entries = payload.get("items")
+    if not isinstance(entries, list):
+        raise RuntimeError("FreshRSS stream response does not contain an items array.")
+
+    mapped_items = [map_entry_to_item(entry).model_dump(mode="json") for entry in entries]
+
+    _save_json(args.raw_output, payload)
+    _save_json(args.items_output, mapped_items)
+
+    print(f"Saved {len(mapped_items)} mapped items to {args.items_output}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SRC_ROOT = REPO_ROOT / "src"
+
+if str(SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(SRC_ROOT))
+
+from summary_mcp.filters.engine import load_filter_rules, evaluate_filter_rules
+from summary_mcp.models.document import ExtractedArticle
+from summary_mcp.models.filtering import FilterContext, FilterInput
+from summary_mcp.models.item import Item
+from summary_mcp.models.llm_result import LlmSummaryResult
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+def _save_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run deterministic filter rules against a structured summary result.")
+    parser.add_argument("--summary", type=Path, required=True, help="Structured LLM summary JSON file")
+    parser.add_argument("--extracted", type=Path, default=None, help="Extracted article JSON file")
+    parser.add_argument("--item", type=Path, default=None, help="Normalized item JSON file")
+    parser.add_argument("--context", type=Path, default=None, help="Optional filter context JSON file")
+    parser.add_argument(
+        "--rules",
+        type=Path,
+        default=REPO_ROOT / "configs" / "filter_rules.json",
+        help="Filter rule config JSON file",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "filter-decision.json",
+        help="Where to save the filter decision",
+    )
+    args = parser.parse_args()
+
+    summary = LlmSummaryResult.model_validate(_load_json(args.summary))
+
+    article = None
+    if args.extracted is not None:
+        extracted_payload = _load_json(args.extracted)
+        article_payload = extracted_payload.get("article", extracted_payload)
+        article = ExtractedArticle.model_validate(article_payload)
+
+    item = None
+    if args.item is not None:
+        item = Item.model_validate(_load_json(args.item))
+
+    context = FilterContext.model_validate(_load_json(args.context)) if args.context else FilterContext()
+    rules = load_filter_rules(args.rules)
+    decision = evaluate_filter_rules(
+        FilterInput(item=item, article=article, summary=summary, context=context),
+        rules,
+    )
+
+    _save_json(args.output, decision.model_dump(mode="json"))
+    print(f"Saved filter decision to {args.output}")
+
+
+if __name__ == "__main__":
+    main()
+
@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SRC_ROOT = REPO_ROOT / "src"
+
+if str(SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(SRC_ROOT))
+
+from summary_mcp.core.pipeline import extract_content
+from summary_mcp.integrations.freshrss import FreshRSSClient, map_entry_to_item
+from summary_mcp.models.summary_io import ExtractionInput
+
+
+def _save_json(path: Path, payload: dict[str, Any] | list[Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _load_required(name: str, value: str | None) -> str:
+    resolved = value or os.environ.get(name)
+    if not resolved:
+        cli_name = name.lower().replace("_", "-")
+        raise RuntimeError(f"Missing required value: pass --{cli_name} or set {name}.")
+    return resolved
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Pull FreshRSS entries and run content extraction for each mapped item.")
+    parser.add_argument("--api-base-url", type=str, default=None, help="FreshRSS greader API base URL")
+    parser.add_argument("--username", type=str, default=None, help="FreshRSS API username")
+    parser.add_argument("--api-password", type=str, default=None, help="FreshRSS API password")
+    parser.add_argument(
+        "--stream-id",
+        type=str,
+        default="user/-/state/com.google/reading-list",
+        help="Google Reader API stream id",
+    )
+    parser.add_argument("--limit", type=int, default=5, help="Maximum number of entries to request")
+    parser.add_argument("--continuation", type=str, default=None, help="Continuation token for paging")
+    parser.add_argument(
+        "--raw-output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "freshrss.raw.json",
+        help="Where to save the raw FreshRSS response",
+    )
+    parser.add_argument(
+        "--items-output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "freshrss.items.json",
+        help="Where to save the mapped item list",
+    )
+    parser.add_argument(
+        "--extracted-output",
+        type=Path,
+        default=REPO_ROOT / "outputs" / "freshrss.extracted.json",
+        help="Where to save the extraction results",
+    )
+    parser.add_argument("--timeout", type=float, default=20.0, help="Request timeout in seconds")
+    args = parser.parse_args()
+
+    api_base_url = _load_required("FRESHRSS_API_BASE_URL", args.api_base_url)
+    username = _load_required("FRESHRSS_USERNAME", args.username)
+    api_password = _load_required("FRESHRSS_API_PASSWORD", args.api_password)
+
+    client = FreshRSSClient(
+        api_base_url=api_base_url,
+        username=username,
+        api_password=api_password,
+        timeout_seconds=args.timeout,
+    )
+    auth_token = client.client_login()
+    payload = client.fetch_stream_contents(
+        auth_token=auth_token,
+        stream_id=args.stream_id,
+        limit=args.limit,
+        continuation=args.continuation,
+    )
+
+    entries = payload.get("items")
+    if not isinstance(entries, list):
+        raise RuntimeError("FreshRSS stream response does not contain an items array.")
+
+    mapped_items = [map_entry_to_item(entry) for entry in entries]
+    extraction_results: list[dict[str, Any]] = []
+    success_count = 0
+
+    for item in mapped_items:
+        extraction = extract_content(ExtractionInput(item=item))
+        extraction_results.append(
+            {
+                "item": item.model_dump(mode="json"),
+                "extraction": extraction.model_dump(mode="json"),
+            }
+        )
+        if extraction.success:
+            success_count += 1
+
+    _save_json(args.raw_output, payload)
+    _save_json(args.items_output, [item.model_dump(mode="json") for item in mapped_items])
+    _save_json(
+        args.extracted_output,
+        {
+            "stream_id": args.stream_id,
+            "requested_limit": args.limit,
+            "entry_count": len(entries),
+            "extracted_success_count": success_count,
+            "results": extraction_results,
+        },
+    )
+
+    print(
+        f"Saved {len(mapped_items)} mapped items and {success_count} successful extractions to {args.extracted_output}"
+    )
+
+
+if __name__ == "__main__":
+    main()