Refine OpenClaw payloads and reorganize docs

2026-03-26 10:20:07 +08:00
parent cecf4d3ae7
commit 27fe1e8882
155 changed files with 8999 additions and 87 deletions
@@ -1 +1,55 @@
-"""Shared models for the summary MCP service."""
+"""Shared models for the summary MCP service."""
+
+from .article_candidate import (
+    ArticleCandidate,
+    ArticleCandidateRecord,
+    CandidateMetadata,
+    CandidateSourceRefs,
+    DigestSectionHint,
+    OpenClawCandidateInput,
+    ReviewState,
+    build_article_candidate_record,
+    build_openclaw_candidate_input,
+    candidate_id_for,
+    normalize_candidate_url,
+)
+from .candidate_review import CandidateReviewStatus, KnowledgeDecision, ReviewStatus
+from .daily_digest import (
+    DailyDigest,
+    DailyDigestItem,
+    DailyDigestMetadata,
+    DailyDigestSection,
+    DailyDigestSourceRef,
+    DailyDigestStats,
+)
+from .openclaw_delivery import (
+    OpenClawDeliveryPayload,
+    OpenClawDeliveryStats,
+    build_openclaw_delivery_payload,
+)
+
+__all__ = [
+    "ArticleCandidate",
+    "ArticleCandidateRecord",
+    "CandidateMetadata",
+    "CandidateReviewStatus",
+    "CandidateSourceRefs",
+    "DailyDigest",
+    "DailyDigestItem",
+    "DailyDigestMetadata",
+    "DailyDigestSection",
+    "DailyDigestSourceRef",
+    "DailyDigestStats",
+    "DigestSectionHint",
+    "KnowledgeDecision",
+    "OpenClawCandidateInput",
+    "OpenClawDeliveryPayload",
+    "OpenClawDeliveryStats",
+    "ReviewState",
+    "ReviewStatus",
+    "build_article_candidate_record",
+    "build_openclaw_delivery_payload",
+    "build_openclaw_candidate_input",
+    "candidate_id_for",
+    "normalize_candidate_url",
+]
@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+
+from pydantic import BaseModel, Field, HttpUrl
+
+from .document import ExtractedArticle
+from .filtering import FilterDecision, FilterDecisionResult
+from .item import Item
+from .llm_result import Category, LlmSummaryResult
+
+
+ReviewState = Literal["pending", "accepted", "rejected", "deferred"]
+DigestSectionHint = Literal[
+    "top_news",
+    "tools_and_workflows",
+    "risk_and_security",
+    "open_source",
+    "insights",
+    "deep_dive",
+]
+
+_TRACKING_QUERY_PARAMS = {
+    "fbclid",
+    "gclid",
+    "igshid",
+    "mc_cid",
+    "mc_eid",
+    "mkt_tok",
+    "ref",
+    "ref_src",
+    "si",
+    "spm",
+}
+
+
+class CandidateSourceRefs(BaseModel):
+    item_path: str | None = None
+    extracted_path: str | None = None
+    summary_path: str | None = None
+    filter_path: str | None = None
+
+
+class CandidateMetadata(BaseModel):
+    generated_at: datetime | None = None
+    pipeline_version: str = "v1"
+    producer: str | None = None
+    run_id: str | None = None
+
+
+class ArticleCandidateRecord(BaseModel):
+    candidate_id: str
+    item: Item | None = None
+    article: ExtractedArticle
+    summary: LlmSummaryResult
+    filter_result: FilterDecisionResult
+    digest_section_hint: DigestSectionHint | None = None
+    digest_rank: int = Field(default=50, ge=0, le=100)
+    review_state: ReviewState = "pending"
+    source_refs: CandidateSourceRefs = Field(default_factory=CandidateSourceRefs)
+    rendered_markdown: str | None = None
+    metadata: CandidateMetadata = Field(default_factory=CandidateMetadata)
+
+
+class OpenClawCandidateInput(BaseModel):
+    candidate_id: str
+    title: str
+    url: HttpUrl
+    canonical_url: HttpUrl | None = None
+    published_at: datetime | None = None
+    author: str | None = None
+    source_name: str | None = None
+    language: str | None = None
+    summary: str
+    highlights: list[str] = Field(default_factory=list)
+    keywords: list[str] = Field(default_factory=list)
+    topics: list[str] = Field(default_factory=list)
+    category: Category
+    worth_keeping: bool
+    worth_reason: str
+    selection_decision: FilterDecision
+    selection_reason: str | None = None
+    digest_section_hint: DigestSectionHint | None = None
+    digest_rank: int = Field(default=50, ge=0, le=100)
+
+
+# Backward-compatible alias while the rest of the codebase migrates to the new name.
+ArticleCandidate = ArticleCandidateRecord
+
+
+def normalize_candidate_url(url: str) -> str:
+    parsed = urlparse(url)
+    filtered_query = [
+        (key, value)
+        for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+        if key.lower() not in _TRACKING_QUERY_PARAMS and not key.lower().startswith("utm_")
+    ]
+    normalized = parsed._replace(query=urlencode(filtered_query, doseq=True), fragment="")
+    return urlunparse(normalized)
+
+
+def candidate_id_for(item: Item | None, article: ExtractedArticle) -> str:
+    if item is not None and item.item_id:
+        return f"cand:{item.item_id}"
+    if article.item_id:
+        return f"cand:{article.item_id}"
+    return f"cand:{article.extract_id}"
+
+
+def build_article_candidate_record(
+    *,
+    summary: LlmSummaryResult,
+    article: ExtractedArticle,
+    filter_result: FilterDecisionResult,
+    item: Item | None = None,
+    digest_section_hint: DigestSectionHint | None = None,
+    digest_rank: int | None = None,
+    rendered_markdown: str | None = None,
+    source_refs: CandidateSourceRefs | None = None,
+    metadata: CandidateMetadata | None = None,
+) -> ArticleCandidateRecord:
+    return ArticleCandidateRecord(
+        candidate_id=candidate_id_for(item, article),
+        item=item,
+        article=article,
+        summary=summary,
+        filter_result=filter_result,
+        digest_section_hint=digest_section_hint,
+        digest_rank=digest_rank if digest_rank is not None else filter_result.priority,
+        source_refs=source_refs or CandidateSourceRefs(),
+        rendered_markdown=rendered_markdown,
+        metadata=metadata or CandidateMetadata(),
+    )
+
+
+def _source_name_from_item(item: Item | None, article: ExtractedArticle) -> str | None:
+    if item is not None:
+        metadata = item.metadata if isinstance(item.metadata, dict) else {}
+        origin = metadata.get("origin")
+        if isinstance(origin, dict):
+            origin_title = origin.get("title")
+            if isinstance(origin_title, str) and origin_title.strip():
+                return origin_title.strip()
+
+        for key in ("source_title", "feed_title", "site_name", "source_name"):
+            value = metadata.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+
+    candidate_url = str(item.url) if item is not None else str(article.url)
+    hostname = urlparse(candidate_url).netloc
+    return hostname or None
+
+
+def build_openclaw_candidate_input(record: ArticleCandidateRecord) -> OpenClawCandidateInput:
+    item = record.item
+    article = record.article
+    summary = record.summary
+    filter_result = record.filter_result
+
+    title = item.title if item is not None and item.title else summary.title
+    raw_url = str(item.url) if item is not None else str(summary.url)
+    published_at = item.published_at if item is not None else article.published_at
+    author = item.author if item is not None and item.author else article.author
+    language = item.language if item is not None and item.language else article.language
+    selection_reason = filter_result.reasons[0] if filter_result.reasons else None
+
+    return OpenClawCandidateInput(
+        candidate_id=record.candidate_id,
+        title=title,
+        url=raw_url,
+        canonical_url=normalize_candidate_url(raw_url),
+        published_at=published_at,
+        author=author,
+        source_name=_source_name_from_item(item, article),
+        language=language,
+        summary=summary.summary,
+        highlights=summary.highlights,
+        keywords=summary.keywords,
+        topics=summary.topics,
+        category=summary.category,
+        worth_keeping=summary.worth_keeping,
+        worth_reason=summary.reason,
+        selection_decision=filter_result.decision,
+        selection_reason=selection_reason,
+        digest_section_hint=record.digest_section_hint,
+        digest_rank=record.digest_rank,
+    )
@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+ReviewStatus = Literal["pending", "confirmed", "rejected", "deferred"]
+KnowledgeDecision = Literal["pending", "ingest", "skip"]
+
+
+class CandidateReviewStatus(BaseModel):
+    candidate_id: str
+    review_status: ReviewStatus = "pending"
+    reviewed_at: datetime | None = None
+    review_note: str | None = None
+    knowledge_decision: KnowledgeDecision = "pending"
+    knowledge_note: str | None = None
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from datetime import date, datetime
+
+from pydantic import BaseModel, Field, HttpUrl
+
+from .article_candidate import DigestSectionHint
+
+
+class DailyDigestItem(BaseModel):
+    candidate_id: str
+    title: str
+    summary: str | None = None
+    why_it_matters: str
+    action: str | None = None
+
+
+class DailyDigestSection(BaseModel):
+    section_id: DigestSectionHint
+    title: str
+    summary: str
+    items: list[DailyDigestItem] = Field(default_factory=list)
+
+
+class DailyDigestSourceRef(BaseModel):
+    candidate_id: str
+    url: HttpUrl | None = None
+    title: str | None = None
+
+
+class DailyDigestStats(BaseModel):
+    candidate_total: int = Field(default=0, ge=0)
+    kept_total: int = Field(default=0, ge=0)
+    review_total: int = Field(default=0, ge=0)
+    dropped_total: int = Field(default=0, ge=0)
+
+
+class DailyDigestMetadata(BaseModel):
+    generated_at: datetime | None = None
+    producer: str | None = None
+    pipeline_version: str = "v1"
+    run_id: str | None = None
+
+
+class DailyDigest(BaseModel):
+    digest_id: str
+    date: date
+    title: str
+    summary: str
+    sections: list[DailyDigestSection] = Field(default_factory=list)
+    top_items: list[str] = Field(default_factory=list)
+    key_takeaways: list[str] = Field(default_factory=list)
+    watchlist: list[str] = Field(default_factory=list)
+    candidate_ids: list[str] = Field(default_factory=list)
+    source_refs: list[DailyDigestSourceRef] = Field(default_factory=list)
+    editor_notes: str | None = None
+    stats: DailyDigestStats = Field(default_factory=DailyDigestStats)
+    metadata: DailyDigestMetadata = Field(default_factory=DailyDigestMetadata)
@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from datetime import UTC, date, datetime
+
+from pydantic import BaseModel, Field
+
+from .article_candidate import OpenClawCandidateInput
+
+
+class OpenClawDeliveryStats(BaseModel):
+    total: int = Field(default=0, ge=0)
+    keep_total: int = Field(default=0, ge=0)
+    review_total: int = Field(default=0, ge=0)
+    drop_total: int = Field(default=0, ge=0)
+
+
+class OpenClawDeliveryPayload(BaseModel):
+    schema_version: str = "v1"
+    generated_at: datetime
+    run_id: str
+    date: date
+    candidates: list[OpenClawCandidateInput] = Field(default_factory=list)
+    stats: OpenClawDeliveryStats = Field(default_factory=OpenClawDeliveryStats)
+
+
+def build_openclaw_delivery_payload(
+    candidates: list[OpenClawCandidateInput],
+    *,
+    run_id: str | None = None,
+    for_date: date | None = None,
+) -> OpenClawDeliveryPayload:
+    now = datetime.now(tz=UTC)
+    payload_date = for_date or now.date()
+    resolved_run_id = run_id or now.strftime("openclaw-delivery-%Y%m%d-%H%M%S")
+
+    stats = OpenClawDeliveryStats(
+        total=len(candidates),
+        keep_total=sum(1 for candidate in candidates if candidate.selection_decision == "keep"),
+        review_total=sum(1 for candidate in candidates if candidate.selection_decision == "review"),
+        drop_total=sum(1 for candidate in candidates if candidate.selection_decision == "drop"),
+    )
+    return OpenClawDeliveryPayload(
+        schema_version="v1",
+        generated_at=now,
+        run_id=resolved_run_id,
+        date=payload_date,
+        candidates=candidates,
+        stats=stats,
+    )