42 lines
1.7 KiB
Python
42 lines
1.7 KiB
Python
"""Spec dispatch: run one or many fetch specs, never raising for a single failure."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Callable
|
|
|
|
from web_fetcher.adapters import eurostat, http
|
|
from web_fetcher.core import Fetched, FetchError, Observation, fetch_bytes
|
|
|
|
ADAPTERS: dict[str, Callable[[dict[str, Any], Callable[[str], Fetched]], list[Observation]]] = {
|
|
"eurostat": eurostat.fetch,
|
|
"http_json": http.fetch_json,
|
|
"http_csv": http.fetch_csv,
|
|
"xlsx_wide": http.fetch_xlsx_wide,
|
|
}
|
|
|
|
|
|
def run_spec(spec: dict[str, Any], get: Callable[[str], Fetched] | None = None,
|
|
cache_dir: str | None = None) -> list[Observation]:
|
|
"""Run one spec. Adapter errors become an `error` observation, not an exception."""
|
|
sid = spec.get("id", "?")
|
|
kind = spec.get("adapter")
|
|
if kind == "manual":
|
|
return [Observation(spec_id=sid, status="not_found", source=spec.get("source"),
|
|
note="manual: no machine-readable source; collect at review")]
|
|
adapter = ADAPTERS.get(kind or "")
|
|
if adapter is None:
|
|
return [Observation(spec_id=sid, status="error", note=f"unknown adapter {kind!r}")]
|
|
getter = get or (lambda url: fetch_bytes(url, cache_dir=cache_dir))
|
|
try:
|
|
return adapter(spec, getter)
|
|
except FetchError as exc:
|
|
return [Observation(spec_id=sid, status="error", note=str(exc)[:300])]
|
|
except (KeyError, ValueError, TypeError) as exc:
|
|
return [Observation(spec_id=sid, status="error", note=f"parse error: {exc!s}"[:300])]
|
|
|
|
|
|
def run_specs(specs: list[dict[str, Any]], **kw: Any) -> list[Observation]:
|
|
out: list[Observation] = []
|
|
for spec in specs:
|
|
out.extend(run_spec(spec, **kw))
|
|
return out
|