97 lines
4.5 KiB
Python
97 lines
4.5 KiB
Python
r"""Generic HTTP adapters: JSON (dotted path) and CSV (filter + pick latest).
|
|
|
|
json spec fields: url, path ("a.b.0.c"), unit?, period_path?
|
|
csv spec fields: url, filter (dict column -> value), value_column,
|
|
period_column?, unit?, delimiter? -> latest row by period
|
|
xlsx_wide fields: url, sheet, filter (dict header -> value), period_regex
|
|
(header pattern of period columns, default ^\d{6}$), unit?
|
|
-> value of the matching row in the latest period column
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import json
|
|
from typing import Any, Callable
|
|
|
|
from web_fetcher.core import Fetched, Observation
|
|
|
|
|
|
def _dig(obj: Any, path: str) -> Any:
|
|
for part in path.split("."):
|
|
if obj is None:
|
|
return None
|
|
if isinstance(obj, list):
|
|
try:
|
|
obj = obj[int(part)]
|
|
except (ValueError, IndexError):
|
|
return None
|
|
else:
|
|
obj = obj.get(part) if isinstance(obj, dict) else None
|
|
return obj
|
|
|
|
|
|
def fetch_json(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
|
|
f = get(spec["url"])
|
|
doc = json.loads(f.body)
|
|
value = _dig(doc, spec["path"])
|
|
period = _dig(doc, spec["period_path"]) if spec.get("period_path") else spec.get("period")
|
|
status = "ok" if value is not None else "not_found"
|
|
return [Observation(spec_id=spec["id"], status=status, value=value, unit=spec.get("unit"),
|
|
period=None if period is None else str(period), source=spec.get("source"),
|
|
url=f.url, retrieved_at=f.retrieved_at, payload_sha256=f.sha256,
|
|
note=None if value is not None else f"path {spec['path']} not found")]
|
|
|
|
|
|
def fetch_csv(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
|
|
f = get(spec["url"])
|
|
text = f.body.decode(spec.get("encoding", "utf-8-sig"), errors="replace")
|
|
rows = list(csv.DictReader(io.StringIO(text), delimiter=spec.get("delimiter", ",")))
|
|
flt = spec.get("filter") or {}
|
|
rows = [r for r in rows if all(str(r.get(k, "")).strip() == str(v) for k, v in flt.items())]
|
|
pcol, vcol = spec.get("period_column"), spec["value_column"]
|
|
rows = [r for r in rows if str(r.get(vcol, "")).strip() not in ("", "NA", "N/A", "-")]
|
|
if not rows:
|
|
return [Observation(spec_id=spec["id"], status="not_found", url=f.url, retrieved_at=f.retrieved_at,
|
|
payload_sha256=f.sha256, source=spec.get("source"), note="no matching row")]
|
|
row = max(rows, key=lambda r: str(r.get(pcol, ""))) if pcol else rows[-1]
|
|
raw = str(row[vcol]).strip().replace("%", "")
|
|
try:
|
|
value: Any = float(raw)
|
|
except ValueError:
|
|
value = raw
|
|
return [Observation(spec_id=spec["id"], status="ok", value=value, unit=spec.get("unit"),
|
|
period=str(row.get(pcol)) if pcol else None, source=spec.get("source"),
|
|
labels={k: str(v) for k, v in flt.items()}, url=f.url,
|
|
retrieved_at=f.retrieved_at, payload_sha256=f.sha256)]
|
|
|
|
|
|
def fetch_xlsx_wide(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
|
|
import re
|
|
|
|
from web_fetcher.xlsx import read_sheet
|
|
|
|
f = get(spec["url"])
|
|
rows = read_sheet(f.body, spec.get("sheet", 0))
|
|
header, body = rows[0], rows[1:]
|
|
pat = re.compile(spec.get("period_regex", r"^\d{6}$"))
|
|
periods = [(i, h) for i, h in enumerate(header) if pat.match(h or "")]
|
|
flt = spec.get("filter") or {}
|
|
col = {h: i for i, h in enumerate(header)}
|
|
match = [r for r in body if all(i < len(r) and r[i].strip() == str(v) for h, v in flt.items()
|
|
for i in [col.get(h, 10**9)])]
|
|
base = dict(spec_id=spec["id"], url=f.url, retrieved_at=f.retrieved_at, payload_sha256=f.sha256,
|
|
source=spec.get("source"), labels={k: str(v) for k, v in flt.items()})
|
|
if not match or not periods:
|
|
return [Observation(status="not_found", note="no matching row or period columns", **base)]
|
|
row = match[0]
|
|
for i, h in sorted(periods, key=lambda p: p[1], reverse=True):
|
|
raw = (row[i] if i < len(row) else "").strip().replace("%", "")
|
|
if raw not in ("", "S", "NA", "-"):
|
|
try:
|
|
value: Any = float(raw)
|
|
except ValueError:
|
|
value = raw
|
|
return [Observation(status="ok", value=value, unit=spec.get("unit"), period=h, **base)]
|
|
return [Observation(status="not_found", note="matching row has no value", **base)]
|