web-fetcher/build/lib/web_fetcher/adapters/http.py

r"""Generic HTTP adapters: JSON (dotted path) and CSV (filter + pick latest).

json spec fields:  url, path ("a.b.0.c"), unit?, period_path?
csv spec fields:   url, filter (dict column -> value), value_column,
                   period_column?, period_format?, value_type?, unit?, delimiter?
                   -> latest row by period
xlsx_wide fields:  url, sheet, filter (dict header -> value), period_regex
                   (header pattern of period columns, default ^\d{6}$),
                   period_format?, value_type?, unit?
                   -> value of the matching row in the latest period column

All three take `source`; a value without one is turned into an error by the
registry. "Latest" follows `web_fetcher.periods` (period_format, default auto)
and the chosen rule is stated in the observation's note. Values must be
numeric unless value_type is "string"; otherwise the result is not_found.
"""

from __future__ import annotations

import csv
import io
import json
from typing import Any, Callable

from web_fetcher.core import Fetched, Observation
from web_fetcher.periods import order

MISSING = ("", "NA", "N/A", "-", ".", "S")


def _value(raw: str, spec: dict[str, Any]) -> tuple[Any, str | None]:
    """Parse a cell; returns (value, None) or (None, reason) for non-numeric data."""
    text = raw.strip()
    if spec.get("value_type") == "string":
        return text, None
    try:
        return float(text.replace("%", "").replace(",", "")), None
    except ValueError:
        return None, f"non-numeric value {text!r} (set value_type: string to accept)"


def _dig(obj: Any, path: str) -> Any:
    for part in path.split("."):
        if obj is None:
            return None
        if isinstance(obj, list):
            try:
                obj = obj[int(part)]
            except (ValueError, IndexError):
                return None
        else:
            obj = obj.get(part) if isinstance(obj, dict) else None
    return obj


def fetch_json(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
    f = get(spec["url"])
    doc = json.loads(f.body)
    value = _dig(doc, spec["path"])
    period = _dig(doc, spec["period_path"]) if spec.get("period_path") else spec.get("period")
    status = "ok" if value is not None else "not_found"
    return [Observation(spec_id=spec["id"], status=status, value=value, unit=spec.get("unit"),
                        period=None if period is None else str(period), source=spec.get("source"),
                        url=f.url, published=f.published, retrieved_at=f.retrieved_at,
                        payload_sha256=f.sha256,
                        note=None if value is not None else f"path {spec['path']} not found")]


def fetch_csv(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
    f = get(spec["url"])
    text = f.body.decode(spec.get("encoding", "utf-8-sig"), errors="replace")
    rows = list(csv.DictReader(io.StringIO(text), delimiter=spec.get("delimiter", ",")))
    flt = spec.get("filter") or {}
    rows = [r for r in rows if all(str(r.get(k, "")).strip() == str(v) for k, v in flt.items())]
    pcol, vcol = spec.get("period_column"), spec["value_column"]
    rows = [r for r in rows if str(r.get(vcol) or "").strip() not in MISSING]
    base = dict(spec_id=spec["id"], url=f.url, published=f.published, retrieved_at=f.retrieved_at,
                payload_sha256=f.sha256, source=spec.get("source"), unit=spec.get("unit"),
                labels={k: str(v) for k, v in flt.items()})
    if not rows:
        return [Observation(status="not_found", note="no matching row", **base)]
    if pcol:
        fmt, newest = order(sorted({str(r.get(pcol) or "").strip() for r in rows}), spec.get("period_format", "auto"))
        period = newest[0]
        latest = [r for r in rows if str(r.get(pcol) or "").strip() == period]
        if len({str(r[vcol]).strip() for r in latest}) > 1:
            raise ValueError(f"{len(latest)} matching rows with different values for period {period}")
        row, rule = latest[-1], f"latest of {len(newest)} periods by {fmt} order"
    else:
        period, row = None, rows[-1]
        rule = f"no period_column: last of {len(rows)} matching rows"
    value, why = _value(str(row[vcol]), spec)
    if why:
        return [Observation(status="not_found", period=period, note=f"{why}; {rule}", **base)]
    return [Observation(status="ok", value=value, period=period, note=rule, **base)]


def fetch_xlsx_wide(spec: dict[str, Any], get: Callable[[str], Fetched]) -> list[Observation]:
    import re

    from web_fetcher.xlsx import read_sheet

    f = get(spec["url"])
    rows = read_sheet(f.body, spec.get("sheet", 0))
    header, body = rows[0], rows[1:]
    pat = re.compile(spec.get("period_regex", r"^\d{6}$"))
    periods = {(h or "").strip(): i for i, h in enumerate(header) if pat.match(h or "")}
    flt = spec.get("filter") or {}
    col = {h: i for i, h in enumerate(header)}
    match = [r for r in body if all(i < len(r) and r[i].strip() == str(v) for h, v in flt.items()
                                    for i in [col.get(h, 10**9)])]
    base = dict(spec_id=spec["id"], url=f.url, published=f.published, retrieved_at=f.retrieved_at,
                payload_sha256=f.sha256, source=spec.get("source"), unit=spec.get("unit"),
                labels={k: str(v) for k, v in flt.items()})
    if not match or not periods:
        return [Observation(status="not_found", note="no matching row or period columns", **base)]
    if len(match) > 1:
        raise ValueError(f"filter matches {len(match)} rows; expected one")
    row = match[0]
    fmt, newest = order(list(periods), spec.get("period_format", "auto"))
    for n, h in enumerate(newest):
        i = periods[h]
        raw = (row[i] if i < len(row) else "").strip()
        if raw in MISSING:
            continue
        rule = f"latest of {len(newest)} periods by {fmt} order"
        if n:
            rule += f", skipping {n} newer period(s) without a value"
        value, why = _value(raw, spec)
        if why:
            return [Observation(status="not_found", period=h, note=f"{why}; {rule}", **base)]
        return [Observation(status="ok", value=value, period=h, note=rule, **base)]
    return [Observation(status="not_found", note="matching row has no value", **base)]