osiman/reader
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Refine OpenClaw payloads and reorganize docs

Browse Source
This commit is contained in:
zhuyongxin
2026-03-26 10:20:07 +08:00
parent cecf4d3ae7
commit 27fe1e8882
155 changed files with 8999 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_openclaw_delivery.py
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import argparse
import json
import sys
from datetime import date
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
OUTPUT_ROOT = REPO_ROOT / "outputs"
REFERENCE_OUTPUT_ROOT = OUTPUT_ROOT / "reference"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
from summary_mcp.models import OpenClawCandidateInput, build_openclaw_delivery_payload
def _load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def _save_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def _collect_input_paths(inputs: list[Path], input_dir: Path | None, pattern: str) -> list[Path]:
resolved = list(inputs)
if input_dir is not None:
resolved.extend(sorted(input_dir.glob(pattern)))
seen: set[Path] = set()
unique_paths: list[Path] = []
for path in resolved:
normalized = path.resolve()
if normalized in seen:
continue
seen.add(normalized)
unique_paths.append(path)
return unique_paths
def main() -> None:
parser = argparse.ArgumentParser(description="Build an OpenClaw batch delivery payload from slim candidate JSON files.")
parser.add_argument("--inputs", nargs="*", type=Path, default=[], help="One or more OpenClaw candidate input JSON files")
parser.add_argument("--input-dir", type=Path, default=None, help="Directory containing OpenClaw candidate input JSON files")
parser.add_argument(
"--pattern",
type=str,
default="*.openclaw-candidate-input.json",
help="Glob pattern used together with --input-dir",
)
parser.add_argument("--run-id", type=str, default=None, help="Optional delivery run id")
parser.add_argument("--date", type=str, default=None, help="Optional delivery date in YYYY-MM-DD format")
parser.add_argument("--sort-by-rank", action="store_true", help="Sort candidates by digest_rank descending")
parser.add_argument(
"--output",
type=Path,
default=REFERENCE_OUTPUT_ROOT / "candidates" / "openclaw-delivery-payload.json",
help="Where to save the OpenClaw delivery payload",
)
args = parser.parse_args()
input_paths = _collect_input_paths(args.inputs, args.input_dir, args.pattern)
if not input_paths:
raise SystemExit("No candidate input files found. Pass --inputs or --input-dir.")
candidates = [OpenClawCandidateInput.model_validate(_load_json(path)) for path in input_paths]
if args.sort_by_rank:
candidates.sort(key=lambda candidate: candidate.digest_rank, reverse=True)
payload = build_openclaw_delivery_payload(
candidates,
run_id=args.run_id,
for_date=date.fromisoformat(args.date) if args.date else None,
)
_save_json(args.output, payload.model_dump(mode="json"))
print(f"Saved OpenClaw delivery payload with {len(candidates)} candidates to {args.output}")
if __name__ == "__main__":
main()
scripts/pull_freshrss_items.py
+6 -4
View File
@@ -1,4 +1,4 @@
from __future__ import annotations
from __future__ import annotations
import argparse
import json
@@ -10,6 +10,8 @@ from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
OUTPUT_ROOT = REPO_ROOT / "outputs"
FRESHRSS_OUTPUT_ROOT = OUTPUT_ROOT / "freshrss"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
@@ -46,13 +48,13 @@ def main() -> None:
parser.add_argument(
"--raw-output",
type=Path,
default=REPO_ROOT / "outputs" / "freshrss.raw.json",
default=FRESHRSS_OUTPUT_ROOT / "raw" / "freshrss.raw.json",
help="Where to save the raw FreshRSS response",
)
parser.add_argument(
"--items-output",
type=Path,
default=REPO_ROOT / "outputs" / "freshrss.items.json",
default=FRESHRSS_OUTPUT_ROOT / "items" / "freshrss.items.json",
help="Where to save the mapped item list",
)
parser.add_argument("--timeout", type=float, default=20.0, help="Request timeout in seconds")
@@ -89,4 +91,4 @@ def main() -> None:
if __name__ == "__main__":
main()
main()
scripts/run_article_candidate.py
+124
View File
@@ -0,0 +1,124 @@
from __future__ import annotations
import argparse
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
OUTPUT_ROOT = REPO_ROOT / "outputs"
REFERENCE_OUTPUT_ROOT = OUTPUT_ROOT / "reference"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
from summary_mcp.models.article_candidate import (
CandidateMetadata,
CandidateSourceRefs,
build_article_candidate_record,
build_openclaw_candidate_input,
)
from summary_mcp.models.document import ExtractedArticle
from summary_mcp.models.filtering import FilterDecisionResult
from summary_mcp.models.item import Item
from summary_mcp.models.llm_result import LlmSummaryResult
def _load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def _save_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser(
description="Build an internal ArticleCandidateRecord and a slim OpenClawCandidateInput from pipeline outputs."
)
parser.add_argument("--summary", type=Path, required=True, help="Structured LLM summary JSON file")
parser.add_argument("--extracted", type=Path, required=True, help="Extracted article JSON file")
parser.add_argument("--filter", type=Path, required=True, help="Filter decision JSON file")
parser.add_argument("--item", type=Path, default=None, help="Optional normalized item JSON file")
parser.add_argument(
"--section-hint",
default=None,
choices=[
"top_news",
"tools_and_workflows",
"risk_and_security",
"open_source",
"insights",
"deep_dive",
],
help="Optional digest section hint for downstream aggregation",
)
parser.add_argument(
"--rank",
type=int,
default=None,
help="Optional digest rank override. Defaults to filter priority when omitted.",
)
parser.add_argument(
"--rendered-markdown",
type=Path,
default=None,
help="Optional markdown file to embed as rendered_markdown",
)
parser.add_argument(
"--output",
dest="record_output",
type=Path,
default=REFERENCE_OUTPUT_ROOT / "candidates" / "article-candidate-record.json",
help="Where to save the internal ArticleCandidateRecord payload",
)
parser.add_argument(
"--openclaw-output",
type=Path,
default=REFERENCE_OUTPUT_ROOT / "candidates" / "openclaw-candidate-input.json",
help="Where to save the slim OpenClawCandidateInput payload",
)
args = parser.parse_args()
summary = LlmSummaryResult.model_validate(_load_json(args.summary))
extracted_payload = _load_json(args.extracted)
article = ExtractedArticle.model_validate(extracted_payload.get("article", extracted_payload))
decision = FilterDecisionResult.model_validate(_load_json(args.filter))
item = Item.model_validate(_load_json(args.item)) if args.item else None
rendered_markdown = args.rendered_markdown.read_text(encoding="utf-8-sig") if args.rendered_markdown else None
record = build_article_candidate_record(
summary=summary,
article=article,
filter_result=decision,
item=item,
digest_section_hint=args.section_hint,
digest_rank=args.rank,
rendered_markdown=rendered_markdown,
source_refs=CandidateSourceRefs(
item_path=str(args.item) if args.item else None,
extracted_path=str(args.extracted),
summary_path=str(args.summary),
filter_path=str(args.filter),
),
metadata=CandidateMetadata(
generated_at=datetime.now(tz=UTC),
producer="run_article_candidate.py",
run_id=datetime.now(tz=UTC).strftime("candidate-%Y%m%d-%H%M%S"),
),
)
openclaw_input = build_openclaw_candidate_input(record)
_save_json(args.record_output, record.model_dump(mode="json"))
_save_json(args.openclaw_output, openclaw_input.model_dump(mode="json"))
print(f"Saved article candidate record to {args.record_output}")
print(f"Saved OpenClaw candidate input to {args.openclaw_output}")
if __name__ == "__main__":
main()
scripts/run_filter_rules.py
+5 -4
View File
@@ -1,4 +1,4 @@
from __future__ import annotations
from __future__ import annotations
import argparse
import json
@@ -9,6 +9,8 @@ from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
OUTPUT_ROOT = REPO_ROOT / "outputs"
REFERENCE_OUTPUT_ROOT = OUTPUT_ROOT / "reference"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
@@ -44,7 +46,7 @@ def main() -> None:
parser.add_argument(
"--output",
type=Path,
default=REPO_ROOT / "outputs" / "filter-decision.json",
default=REFERENCE_OUTPUT_ROOT / "filter" / "filter-decision.json",
help="Where to save the filter decision",
)
args = parser.parse_args()
@@ -73,5 +75,4 @@ def main() -> None:
if __name__ == "__main__":
main()
main()
scripts/run_freshrss_extract.py
+7 -5
View File
@@ -1,4 +1,4 @@
from __future__ import annotations
from __future__ import annotations
import argparse
import json
@@ -10,6 +10,8 @@ from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
SRC_ROOT = REPO_ROOT / "src"
OUTPUT_ROOT = REPO_ROOT / "outputs"
FRESHRSS_OUTPUT_ROOT = OUTPUT_ROOT / "freshrss"
if str(SRC_ROOT) not in sys.path:
sys.path.insert(0, str(SRC_ROOT))
@@ -48,19 +50,19 @@ def main() -> None:
parser.add_argument(
"--raw-output",
type=Path,
default=REPO_ROOT / "outputs" / "freshrss.raw.json",
default=FRESHRSS_OUTPUT_ROOT / "raw" / "freshrss.raw.json",
help="Where to save the raw FreshRSS response",
)
parser.add_argument(
"--items-output",
type=Path,
default=REPO_ROOT / "outputs" / "freshrss.items.json",
default=FRESHRSS_OUTPUT_ROOT / "items" / "freshrss.items.json",
help="Where to save the mapped item list",
)
parser.add_argument(
"--extracted-output",
type=Path,
default=REPO_ROOT / "outputs" / "freshrss.extracted.json",
default=FRESHRSS_OUTPUT_ROOT / "extracted" / "freshrss.extracted.json",
help="Where to save the extraction results",
)
parser.add_argument("--timeout", type=float, default=20.0, help="Request timeout in seconds")
@@ -122,4 +124,4 @@ def main() -> None:
if __name__ == "__main__":
main()
main()