tegwick
96fe5b022b
- --replay DIR / replay_dir: offline replay from the cache, hash-verified - ok values without source become error; Last-Modified recorded as published - latest period by parsed order (web_fetcher/periods.py); mixed formats refused - non-numeric values not_found unless value_type: string - retry only connection errors, timeouts, 429/5xx; honour Retry-After - opt-in live smoke test; 18 offline tests - CHANGELOG, README install docs, SCOPE update, capability D2/A3/C2/R2 - WEBFETCH-WP-0003 (repo hygiene) added Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 153021@bnt-lap001 Assistant-Session: 46917be4-6400-4106-b8b7-61953a0caf24
54 lines
2.3 KiB
Python
54 lines
2.3 KiB
Python
"""Spec dispatch: run one or many fetch specs, never raising for a single failure."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Callable
|
|
|
|
from web_fetcher.adapters import eurostat, http
|
|
from web_fetcher.core import Fetched, FetchError, Observation, fetch_bytes
|
|
|
|
ADAPTERS: dict[str, Callable[[dict[str, Any], Callable[[str], Fetched]], list[Observation]]] = {
|
|
"eurostat": eurostat.fetch,
|
|
"http_json": http.fetch_json,
|
|
"http_csv": http.fetch_csv,
|
|
"xlsx_wide": http.fetch_xlsx_wide,
|
|
}
|
|
|
|
|
|
def run_spec(spec: dict[str, Any], get: Callable[[str], Fetched] | None = None,
|
|
cache_dir: str | None = None, replay_dir: str | None = None) -> list[Observation]:
|
|
"""Run one spec. Adapter errors become an `error` observation, not an exception.
|
|
|
|
A value is never returned without a source: an `ok` observation whose
|
|
adapter could not name one (the spec lacks `source`) becomes an `error`.
|
|
"""
|
|
sid = spec.get("id", "?")
|
|
kind = spec.get("adapter")
|
|
if kind == "manual":
|
|
return [Observation(spec_id=sid, status="not_found", source=spec.get("source"),
|
|
note="manual: no machine-readable source; collect at review")]
|
|
adapter = ADAPTERS.get(kind or "")
|
|
if adapter is None:
|
|
return [Observation(spec_id=sid, status="error", note=f"unknown adapter {kind!r}")]
|
|
getter = get or (lambda url: fetch_bytes(url, cache_dir=cache_dir, replay_dir=replay_dir))
|
|
try:
|
|
return [_require_source(o) for o in adapter(spec, getter)]
|
|
except FetchError as exc:
|
|
return [Observation(spec_id=sid, status="error", note=str(exc)[:300])]
|
|
except (KeyError, ValueError, TypeError) as exc:
|
|
return [Observation(spec_id=sid, status="error", note=f"parse error: {exc!s}"[:300])]
|
|
|
|
|
|
def _require_source(obs: Observation) -> Observation:
|
|
if obs.status == "ok" and not obs.source:
|
|
return Observation(spec_id=obs.spec_id, status="error", url=obs.url, retrieved_at=obs.retrieved_at,
|
|
payload_sha256=obs.payload_sha256,
|
|
note="spec has no 'source'; values are never returned without one")
|
|
return obs
|
|
|
|
|
|
def run_specs(specs: list[dict[str, Any]], **kw: Any) -> list[Observation]:
|
|
out: list[Observation] = []
|
|
for spec in specs:
|
|
out.extend(run_spec(spec, **kw))
|
|
return out
|