fix: fallback to repo .env for reader pipeline settings
This commit is contained in:
@@ -10,6 +10,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
from dotenv import dotenv_values
|
||||||
|
|
||||||
from summary_mcp.validators.llm_result import ValidationReport
|
from summary_mcp.validators.llm_result import ValidationReport
|
||||||
from summary_mcp.validators.llm_result import validate_llm_result as validate_llm_result_from_path
|
from summary_mcp.validators.llm_result import validate_llm_result as validate_llm_result_from_path
|
||||||
@@ -18,6 +19,8 @@ from summary_mcp.validators.llm_result import validate_llm_result_payload
|
|||||||
|
|
||||||
JSON_BLOCK_RE = re.compile(r"```(?:json)?\s*(\{.*\})\s*```", re.DOTALL)
|
JSON_BLOCK_RE = re.compile(r"```(?:json)?\s*(\{.*\})\s*```", re.DOTALL)
|
||||||
DEFAULT_CHAT_COMPLETIONS_URL = "https://api.openai.com/v1/chat/completions"
|
DEFAULT_CHAT_COMPLETIONS_URL = "https://api.openai.com/v1/chat/completions"
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
DEFAULT_DOTENV_PATH = REPO_ROOT / ".env"
|
||||||
|
|
||||||
|
|
||||||
def load_text(path: Path) -> str:
|
def load_text(path: Path) -> str:
|
||||||
@@ -99,22 +102,60 @@ def normalize_chat_completions_url(api_url: str | None) -> str | None:
|
|||||||
return f"{normalized}/chat/completions"
|
return f"{normalized}/chat/completions"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_repo_dotenv() -> dict[str, str]:
|
||||||
|
if not DEFAULT_DOTENV_PATH.exists():
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
key: value
|
||||||
|
for key, value in dotenv_values(DEFAULT_DOTENV_PATH).items()
|
||||||
|
if isinstance(key, str) and isinstance(value, str) and value
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_value(*values: str | None) -> str | None:
|
||||||
|
for value in values:
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def resolve_llm_settings(
|
def resolve_llm_settings(
|
||||||
*,
|
*,
|
||||||
api_key: str | None = None,
|
api_key: str | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
api_url: str | None = None,
|
api_url: str | None = None,
|
||||||
) -> tuple[str, str, str]:
|
) -> tuple[str, str, str]:
|
||||||
resolved_api_key = api_key or os.environ.get("LLM_API_KEY") or os.environ.get("OPENAI_API_KEY")
|
dotenv_map = _load_repo_dotenv()
|
||||||
|
|
||||||
|
resolved_api_key = _pick_value(
|
||||||
|
api_key,
|
||||||
|
os.environ.get("LLM_API_KEY"),
|
||||||
|
os.environ.get("OPENAI_API_KEY"),
|
||||||
|
dotenv_map.get("LLM_API_KEY"),
|
||||||
|
dotenv_map.get("OPENAI_API_KEY"),
|
||||||
|
)
|
||||||
if not resolved_api_key:
|
if not resolved_api_key:
|
||||||
raise RuntimeError("Missing LLM_API_KEY or OPENAI_API_KEY, or pass an API key.")
|
raise RuntimeError("Missing LLM_API_KEY or OPENAI_API_KEY, or pass an API key.")
|
||||||
|
|
||||||
resolved_model = model or os.environ.get("LLM_MODEL") or os.environ.get("OPENAI_MODEL")
|
resolved_model = _pick_value(
|
||||||
|
model,
|
||||||
|
os.environ.get("LLM_MODEL"),
|
||||||
|
os.environ.get("OPENAI_MODEL"),
|
||||||
|
dotenv_map.get("LLM_MODEL"),
|
||||||
|
dotenv_map.get("OPENAI_MODEL"),
|
||||||
|
)
|
||||||
if not resolved_model:
|
if not resolved_model:
|
||||||
raise RuntimeError("Missing LLM_MODEL or OPENAI_MODEL, or pass a model.")
|
raise RuntimeError("Missing LLM_MODEL or OPENAI_MODEL, or pass a model.")
|
||||||
|
|
||||||
resolved_api_url = normalize_chat_completions_url(
|
resolved_api_url = normalize_chat_completions_url(
|
||||||
api_url or os.environ.get("LLM_API_URL") or os.environ.get("OPENAI_API_URL") or DEFAULT_CHAT_COMPLETIONS_URL
|
_pick_value(
|
||||||
|
api_url,
|
||||||
|
os.environ.get("LLM_API_URL"),
|
||||||
|
os.environ.get("OPENAI_API_URL"),
|
||||||
|
dotenv_map.get("LLM_API_URL"),
|
||||||
|
dotenv_map.get("OPENAI_API_URL"),
|
||||||
|
DEFAULT_CHAT_COMPLETIONS_URL,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if not resolved_api_url:
|
if not resolved_api_url:
|
||||||
raise RuntimeError("Missing LLM_API_URL, OPENAI_API_URL, or pass an API URL.")
|
raise RuntimeError("Missing LLM_API_URL, OPENAI_API_URL, or pass an API URL.")
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
|||||||
from typing import Iterable, Mapping, Sequence
|
from typing import Iterable, Mapping, Sequence
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
from dotenv import dotenv_values
|
||||||
|
|
||||||
from summary_mcp.core.summary_loop import run_loop_payload
|
from summary_mcp.core.summary_loop import run_loop_payload
|
||||||
from summary_mcp.validators.article_summary import validate_article_summary_payload
|
from summary_mcp.validators.article_summary import validate_article_summary_payload
|
||||||
@@ -14,6 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3]
|
|||||||
OUTPUT_ROOT = REPO_ROOT / "outputs"
|
OUTPUT_ROOT = REPO_ROOT / "outputs"
|
||||||
FRESHRSS_OUTPUT_ROOT = OUTPUT_ROOT / "freshrss"
|
FRESHRSS_OUTPUT_ROOT = OUTPUT_ROOT / "freshrss"
|
||||||
DEFAULT_PROMPT_PATH = OUTPUT_ROOT / "prompts" / "article-summary-prompt.txt"
|
DEFAULT_PROMPT_PATH = OUTPUT_ROOT / "prompts" / "article-summary-prompt.txt"
|
||||||
|
DEFAULT_DOTENV_PATH = REPO_ROOT / ".env"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -37,6 +39,7 @@ def _resolve_article_llm_settings(
|
|||||||
Resolution order for each field:
|
Resolution order for each field:
|
||||||
- explicit function argument
|
- explicit function argument
|
||||||
- ARTICLE_SUMMARY_* environment variable
|
- ARTICLE_SUMMARY_* environment variable
|
||||||
|
- ARTICLE_SUMMARY_* in repo .env
|
||||||
- main LLM_* / OPENAI_* environment variables (handled by summary_loop.resolve_llm_settings)
|
- main LLM_* / OPENAI_* environment variables (handled by summary_loop.resolve_llm_settings)
|
||||||
|
|
||||||
This helper intentionally does not validate presence; the summary loop
|
This helper intentionally does not validate presence; the summary loop
|
||||||
@@ -45,9 +48,17 @@ def _resolve_article_llm_settings(
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
resolved_api_key = api_key or os.environ.get("ARTICLE_SUMMARY_LLM_API_KEY")
|
dotenv_map = {}
|
||||||
resolved_model = model or os.environ.get("ARTICLE_SUMMARY_LLM_MODEL")
|
if DEFAULT_DOTENV_PATH.exists():
|
||||||
resolved_api_url = api_url or os.environ.get("ARTICLE_SUMMARY_LLM_API_URL")
|
dotenv_map = {
|
||||||
|
key: value
|
||||||
|
for key, value in dotenv_values(DEFAULT_DOTENV_PATH).items()
|
||||||
|
if isinstance(key, str) and isinstance(value, str) and value
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved_api_key = api_key or os.environ.get("ARTICLE_SUMMARY_LLM_API_KEY") or dotenv_map.get("ARTICLE_SUMMARY_LLM_API_KEY")
|
||||||
|
resolved_model = model or os.environ.get("ARTICLE_SUMMARY_LLM_MODEL") or dotenv_map.get("ARTICLE_SUMMARY_LLM_MODEL")
|
||||||
|
resolved_api_url = api_url or os.environ.get("ARTICLE_SUMMARY_LLM_API_URL") or dotenv_map.get("ARTICLE_SUMMARY_LLM_API_URL")
|
||||||
return resolved_api_key, resolved_model, resolved_api_url
|
return resolved_api_key, resolved_model, resolved_api_url
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ from datetime import date, datetime, timezone
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from dotenv import dotenv_values
|
||||||
|
|
||||||
from summary_mcp.core.keyword_index import persist_keyword_indexes
|
from summary_mcp.core.keyword_index import persist_keyword_indexes
|
||||||
from summary_mcp.core.pipeline import extract_content
|
from summary_mcp.core.pipeline import extract_content
|
||||||
from summary_mcp.core.summary_loop import resolve_llm_settings, run_loop_payload
|
from summary_mcp.core.summary_loop import resolve_llm_settings, run_loop_payload
|
||||||
@@ -40,6 +42,7 @@ DEFAULT_TERM_ALIASES_PATH = REPO_ROOT / "configs" / "term_aliases.json"
|
|||||||
DEFAULT_TERM_STOPWORDS_PATH = REPO_ROOT / "configs" / "term_stopwords.json"
|
DEFAULT_TERM_STOPWORDS_PATH = REPO_ROOT / "configs" / "term_stopwords.json"
|
||||||
DEFAULT_TERM_DAILY_DIR = DATA_ROOT / "daily"
|
DEFAULT_TERM_DAILY_DIR = DATA_ROOT / "daily"
|
||||||
DEFAULT_TERM_STATS_PATH = DATA_ROOT / "term_stats.json"
|
DEFAULT_TERM_STATS_PATH = DATA_ROOT / "term_stats.json"
|
||||||
|
DEFAULT_DOTENV_PATH = REPO_ROOT / ".env"
|
||||||
WORKFLOW_NAME = "freshrss_daily_digest"
|
WORKFLOW_NAME = "freshrss_daily_digest"
|
||||||
RUN_TYPE = "daily_digest"
|
RUN_TYPE = "daily_digest"
|
||||||
FETCH_STAGE = "fetch_feed"
|
FETCH_STAGE = "fetch_feed"
|
||||||
@@ -59,14 +62,27 @@ def _load_json(path: Path) -> dict[str, Any]:
|
|||||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||||
|
|
||||||
|
|
||||||
def _load_required_env(name: str, value: str | None) -> str:
|
def _load_repo_dotenv() -> dict[str, str]:
|
||||||
|
if not DEFAULT_DOTENV_PATH.exists():
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
key: value
|
||||||
|
for key, value in dotenv_values(DEFAULT_DOTENV_PATH).items()
|
||||||
|
if isinstance(key, str) and isinstance(value, str) and value
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _load_required_env(name: str, value: str | None, dotenv_map: dict[str, str] | None = None) -> str:
|
||||||
if value:
|
if value:
|
||||||
return value
|
return value
|
||||||
env_value = os.environ.get(name)
|
env_value = os.environ.get(name)
|
||||||
if env_value:
|
if env_value:
|
||||||
return env_value
|
return env_value
|
||||||
|
dotenv_value = (dotenv_map or {}).get(name)
|
||||||
|
if dotenv_value:
|
||||||
|
return dotenv_value
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Missing required value '{name}': not passed as argument and not set as environment variable."
|
f"Missing required value '{name}': not passed as argument, not set as environment variable, and not found in {DEFAULT_DOTENV_PATH}."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -243,9 +259,10 @@ def run_freshrss_pipeline(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
run_store.start_stage(FETCH_STAGE, outputs={"output_dir": str(resolved_output_dir)})
|
run_store.start_stage(FETCH_STAGE, outputs={"output_dir": str(resolved_output_dir)})
|
||||||
resolved_api_base_url = _load_required_env("FRESHRSS_API_BASE_URL", api_base_url)
|
dotenv_map = _load_repo_dotenv()
|
||||||
resolved_username = _load_required_env("FRESHRSS_USERNAME", username)
|
resolved_api_base_url = _load_required_env("FRESHRSS_API_BASE_URL", api_base_url, dotenv_map)
|
||||||
resolved_api_password = _load_required_env("FRESHRSS_API_PASSWORD", api_password)
|
resolved_username = _load_required_env("FRESHRSS_USERNAME", username, dotenv_map)
|
||||||
|
resolved_api_password = _load_required_env("FRESHRSS_API_PASSWORD", api_password, dotenv_map)
|
||||||
resolved_llm_api_key, resolved_llm_model, resolved_llm_api_url = resolve_llm_settings(
|
resolved_llm_api_key, resolved_llm_model, resolved_llm_api_url = resolve_llm_settings(
|
||||||
api_key=llm_api_key,
|
api_key=llm_api_key,
|
||||||
model=llm_model,
|
model=llm_model,
|
||||||
|
|||||||
Reference in New Issue
Block a user