"""Spec dispatch: run one or many fetch specs, never raising for a single failure.""" from __future__ import annotations from typing import Any, Callable from web_fetcher.adapters import eurostat, http from web_fetcher.core import Fetched, FetchError, Observation, fetch_bytes ADAPTERS: dict[str, Callable[[dict[str, Any], Callable[[str], Fetched]], list[Observation]]] = { "eurostat": eurostat.fetch, "http_json": http.fetch_json, "http_csv": http.fetch_csv, "xlsx_wide": http.fetch_xlsx_wide, } def run_spec(spec: dict[str, Any], get: Callable[[str], Fetched] | None = None, cache_dir: str | None = None, replay_dir: str | None = None) -> list[Observation]: """Run one spec. Adapter errors become an `error` observation, not an exception. A value is never returned without a source: an `ok` observation whose adapter could not name one (the spec lacks `source`) becomes an `error`. """ sid = spec.get("id", "?") kind = spec.get("adapter") if kind == "manual": return [Observation(spec_id=sid, status="not_found", source=spec.get("source"), note="manual: no machine-readable source; collect at review")] adapter = ADAPTERS.get(kind or "") if adapter is None: return [Observation(spec_id=sid, status="error", note=f"unknown adapter {kind!r}")] getter = get or (lambda url: fetch_bytes(url, cache_dir=cache_dir, replay_dir=replay_dir)) try: return [_require_source(o) for o in adapter(spec, getter)] except FetchError as exc: return [Observation(spec_id=sid, status="error", note=str(exc)[:300])] except (KeyError, ValueError, TypeError) as exc: return [Observation(spec_id=sid, status="error", note=f"parse error: {exc!s}"[:300])] def _require_source(obs: Observation) -> Observation: if obs.status == "ok" and not obs.source: return Observation(spec_id=obs.spec_id, status="error", url=obs.url, retrieved_at=obs.retrieved_at, payload_sha256=obs.payload_sha256, note="spec has no 'source'; values are never returned without one") return obs def run_specs(specs: list[dict[str, Any]], **kw: Any) -> list[Observation]: out: list[Observation] = [] for spec in specs: out.extend(run_spec(spec, **kw)) return out