"""Spec dispatch: run one or many fetch specs, never raising for a single failure.""" from __future__ import annotations from typing import Any, Callable from web_fetcher.adapters import eurostat, http from web_fetcher.core import Fetched, FetchError, Observation, fetch_bytes ADAPTERS: dict[str, Callable[[dict[str, Any], Callable[[str], Fetched]], list[Observation]]] = { "eurostat": eurostat.fetch, "http_json": http.fetch_json, "http_csv": http.fetch_csv, "xlsx_wide": http.fetch_xlsx_wide, } def run_spec(spec: dict[str, Any], get: Callable[[str], Fetched] | None = None, cache_dir: str | None = None) -> list[Observation]: """Run one spec. Adapter errors become an `error` observation, not an exception.""" sid = spec.get("id", "?") kind = spec.get("adapter") if kind == "manual": return [Observation(spec_id=sid, status="not_found", source=spec.get("source"), note="manual: no machine-readable source; collect at review")] adapter = ADAPTERS.get(kind or "") if adapter is None: return [Observation(spec_id=sid, status="error", note=f"unknown adapter {kind!r}")] getter = get or (lambda url: fetch_bytes(url, cache_dir=cache_dir)) try: return adapter(spec, getter) except FetchError as exc: return [Observation(spec_id=sid, status="error", note=str(exc)[:300])] except (KeyError, ValueError, TypeError) as exc: return [Observation(spec_id=sid, status="error", note=f"parse error: {exc!s}"[:300])] def run_specs(specs: list[dict[str, Any]], **kw: Any) -> list[Observation]: out: list[Observation] = [] for spec in specs: out.extend(run_spec(spec, **kw)) return out