93 lines
3 KiB
Python
93 lines
3 KiB
Python
"""Core types: Observation record and provenance-tracked HTTP fetching."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
USER_AGENT = "web-fetcher/0.1 (+https://forgejo.coulomb.social/coulomb/web-fetcher)"
|
|
|
|
|
|
@dataclass
|
|
class Observation:
|
|
"""One fetched value with its provenance. status: ok | not_found | error."""
|
|
|
|
spec_id: str
|
|
status: str
|
|
value: Any = None
|
|
unit: str | None = None
|
|
period: str | None = None
|
|
labels: dict[str, str] = field(default_factory=dict)
|
|
source: str | None = None # dataset or source name
|
|
url: str | None = None
|
|
published: str | None = None # source-side update/publication timestamp
|
|
retrieved_at: str | None = None
|
|
payload_sha256: str | None = None
|
|
note: str | None = None
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
def to_json(self) -> str:
|
|
return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True)
|
|
|
|
|
|
@dataclass
|
|
class Fetched:
|
|
url: str
|
|
body: bytes
|
|
retrieved_at: str
|
|
sha256: str
|
|
content_type: str | None
|
|
|
|
|
|
class FetchError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def fetch_bytes(url: str, *, timeout: float = 60.0, retries: int = 2,
|
|
cache_dir: str | os.PathLike | None = None,
|
|
headers: dict[str, str] | None = None) -> Fetched:
|
|
"""GET `url` and return body plus provenance. Optional content-addressed cache.
|
|
|
|
The cache stores each payload under its SHA-256 and an index entry per URL,
|
|
so a research run can be replayed offline from the exact bytes it used.
|
|
"""
|
|
hdrs = {"User-Agent": USER_AGENT, **(headers or {})}
|
|
last: Exception | None = None
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
req = urllib.request.Request(url, headers=hdrs)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
body = resp.read()
|
|
ctype = resp.headers.get("Content-Type")
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
fetched = Fetched(url=url, body=body, retrieved_at=_now(), sha256=digest, content_type=ctype)
|
|
if cache_dir:
|
|
_cache_store(Path(cache_dir), fetched)
|
|
return fetched
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
last = exc
|
|
if attempt < retries:
|
|
time.sleep(1.5 * (attempt + 1))
|
|
raise FetchError(f"fetch failed for {url}: {last}")
|
|
|
|
|
|
def _cache_store(cache: Path, fetched: Fetched) -> None:
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
(cache / f"{fetched.sha256}.bin").write_bytes(fetched.body)
|
|
with (cache / "index.jsonl").open("a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps({"url": fetched.url, "sha256": fetched.sha256,
|
|
"retrieved_at": fetched.retrieved_at}) + "\n")
|