tegwick
49de8d15b6
- .forgejo/workflows/test.yaml: offline tests on Python 3.10 and 3.13 - LICENSE: Target Revenue Source License V1C1 (org-wide preliminary licence) - pyproject: license field; AGENTS/SCOPE: CI and licence facts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 153021@bnt-lap001 Assistant-Session: 46917be4-6400-4106-b8b7-61953a0caf24
61 lines
2.6 KiB
Python
61 lines
2.6 KiB
Python
"""Eurostat dissemination API (JSON-stat 2.0).
|
|
|
|
Spec fields:
|
|
dataset Eurostat dataset code, e.g. "isoc_eb_ai"
|
|
filters dict of dimension -> code or list of codes (all non-time dimensions
|
|
should be pinned to one code except those listed in `split_by`)
|
|
split_by dimensions that may vary; one observation per combination (e.g. ["geo"])
|
|
last number of latest time periods to return (default 1)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import itertools
|
|
import json
|
|
import urllib.parse
|
|
from typing import Any, Callable
|
|
|
|
from web_fetcher.core import Fetched, Observation
|
|
|
|
BASE = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/"
|
|
|
|
|
|
def build_url(spec: dict[str, Any]) -> str:
|
|
params: list[tuple[str, str]] = [("format", "JSON"), ("lang", "EN")]
|
|
for dim, val in (spec.get("filters") or {}).items():
|
|
for v in (val if isinstance(val, list) else [val]):
|
|
params.append((dim, str(v)))
|
|
params.append(("lastTimePeriod", str(int(spec.get("last", 1)))))
|
|
return BASE + spec["dataset"] + "?" + urllib.parse.urlencode(params)
|
|
|
|
|
|
def parse(doc: dict[str, Any], spec: dict[str, Any], fetched: Fetched) -> list[Observation]:
|
|
ids = doc["id"]
|
|
dims = doc["dimension"]
|
|
cats = [sorted(dims[i]["category"]["index"], key=lambda k: dims[i]["category"]["index"][k]) for i in ids]
|
|
values = doc.get("value") or {}
|
|
unit_dim = "unit" if "unit" in ids else None
|
|
out: list[Observation] = []
|
|
for n, combo in enumerate(itertools.product(*cats)):
|
|
v = values.get(str(n)) if isinstance(values, dict) else (values[n] if n < len(values) else None)
|
|
if v is None:
|
|
continue
|
|
coord = dict(zip(ids, combo))
|
|
labels = {d: coord[d] for d in spec.get("split_by", [])}
|
|
out.append(Observation(
|
|
spec_id=spec["id"], status="ok", value=v,
|
|
unit=(dims[unit_dim]["category"]["label"].get(coord[unit_dim]) if unit_dim else None),
|
|
period=coord.get("time"), labels=labels, source=f"Eurostat {spec['dataset']}: {doc.get('label')}",
|
|
url=fetched.url, published=doc.get("updated"), retrieved_at=fetched.retrieved_at,
|
|
payload_sha256=fetched.sha256,
|
|
))
|
|
if not out:
|
|
out.append(Observation(spec_id=spec["id"], status="not_found", url=fetched.url,
|
|
retrieved_at=fetched.retrieved_at, payload_sha256=fetched.sha256,
|
|
note="no values for the given filters"))
|
|
return out
|
|
|
|
|
|
def fetch(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
|
|
fetched = get(build_url(spec))
|
|
return parse(json.loads(fetched.body), spec, fetched)
|