Release 0.2.0: replay, source enforcement, period rule, selective retry (WEBFETCH-WP-0002)
- --replay DIR / replay_dir: offline replay from the cache, hash-verified - ok values without source become error; Last-Modified recorded as published - latest period by parsed order (web_fetcher/periods.py); mixed formats refused - non-numeric values not_found unless value_type: string - retry only connection errors, timeouts, 429/5xx; honour Retry-After - opt-in live smoke test; 18 offline tests - CHANGELOG, README install docs, SCOPE update, capability D2/A3/C2/R2 - WEBFETCH-WP-0003 (repo hygiene) added Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 153021@bnt-lap001 Assistant-Session: 46917be4-6400-4106-b8b7-61953a0caf24
This commit is contained in:
parent
5f4467dad8
commit
96fe5b022b
16 changed files with 696 additions and 133 deletions
44
tests/test_live.py
Normal file
44
tests/test_live.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
"""Live smoke tests against real endpoints, one spec per adapter.
|
||||
|
||||
Skipped unless WEB_FETCHER_LIVE=1. They detect upstream shape changes that the
|
||||
recorded fixtures cannot; a failure here means a source changed, not
|
||||
necessarily that web-fetcher is broken.
|
||||
|
||||
WEB_FETCHER_LIVE=1 python3 -B -m unittest tests.test_live -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from web_fetcher.registry import run_spec
|
||||
|
||||
SPECS = [
|
||||
{"id": "live-eurostat", "adapter": "eurostat", "dataset": "isoc_eb_ai",
|
||||
"filters": {"geo": ["EU27_2020"], "unit": "PC_ENT", "size_emp": "GE10", "indic_is": "E_AI_TANY"},
|
||||
"split_by": ["geo"]},
|
||||
{"id": "live-json", "adapter": "http_json",
|
||||
"url": "https://api.worldbank.org/v2/country/DEU/indicator/SE.XPD.TOTL.GD.ZS?format=json&mrnev=1",
|
||||
"path": "1.0.value", "period_path": "1.0.date", "source": "World Bank WDI (SE.XPD.TOTL.GD.ZS)"},
|
||||
{"id": "live-csv", "adapter": "http_csv", "url": "https://fred.stlouisfed.org/graph/fredgraph.csv?id=OPHNFB",
|
||||
"value_column": "OPHNFB", "period_column": "observation_date", "source": "BLS via FRED (OPHNFB)"},
|
||||
{"id": "live-xlsx", "adapter": "xlsx_wide", "url": "https://www.census.gov/hfp/btos/downloads/National.xlsx",
|
||||
"sheet": "Response Estimates", "filter": {"Question ID": "7", "Answer ID": "1"},
|
||||
"source": "U.S. Census Bureau BTOS"},
|
||||
]
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get("WEB_FETCHER_LIVE") == "1", "set WEB_FETCHER_LIVE=1 for live tests")
|
||||
class TestLive(unittest.TestCase):
|
||||
def test_each_adapter_returns_a_sourced_value(self):
|
||||
for spec in SPECS:
|
||||
with self.subTest(spec["id"]):
|
||||
obs = run_spec(spec)
|
||||
self.assertTrue(obs, "no observations")
|
||||
o = obs[0]
|
||||
self.assertEqual(o.status, "ok", o.note)
|
||||
self.assertIsNotNone(o.value)
|
||||
self.assertTrue(o.source and o.url and o.retrieved_at and o.payload_sha256)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -41,7 +41,7 @@ class TestXlsx(unittest.TestCase):
|
|||
body = (FIX / "btos_national.xlsx").read_bytes()
|
||||
obs = run_spec({"id": "us_ai_use", "adapter": "xlsx_wide", "url": "https://x.test/National.xlsx",
|
||||
"sheet": "Response Estimates", "filter": {"Question ID": "7", "Answer ID": "1"},
|
||||
"unit": "% of firms"}, get=fake(body))
|
||||
"unit": "% of firms", "source": "Census BTOS"}, get=fake(body))
|
||||
self.assertEqual(obs[0].status, "ok")
|
||||
self.assertEqual(obs[0].period, "202620")
|
||||
self.assertAlmostEqual(obs[0].value, 23.8)
|
||||
|
|
@ -51,7 +51,7 @@ class TestHttp(unittest.TestCase):
|
|||
def test_json_path(self):
|
||||
body = json.dumps({"data": {"rows": [{"v": 7.5, "p": "2026-Q3"}]}}).encode()
|
||||
obs = run_spec({"id": "j", "adapter": "http_json", "url": "https://x.test", "path": "data.rows.0.v",
|
||||
"period_path": "data.rows.0.p", "unit": "%"}, get=fake(body))
|
||||
"period_path": "data.rows.0.p", "unit": "%", "source": "test"}, get=fake(body))
|
||||
self.assertEqual((obs[0].status, obs[0].value, obs[0].period), ("ok", 7.5, "2026-Q3"))
|
||||
|
||||
def test_json_missing_path_is_not_found(self):
|
||||
|
|
@ -62,7 +62,8 @@ class TestHttp(unittest.TestCase):
|
|||
def test_csv_latest_matching_row(self):
|
||||
body = b"geo,period,value\nUS,2026-08,8.1\nUS,2026-09,8.7\nDE,2026-09,9.9\n"
|
||||
obs = run_spec({"id": "c", "adapter": "http_csv", "url": "https://x.test", "filter": {"geo": "US"},
|
||||
"value_column": "value", "period_column": "period"}, get=fake(body))
|
||||
"value_column": "value", "period_column": "period", "source": "test"},
|
||||
get=fake(body))
|
||||
self.assertEqual((obs[0].value, obs[0].period), (8.7, "2026-09"))
|
||||
|
||||
def test_manual_and_unknown_adapters_never_raise(self):
|
||||
|
|
@ -80,3 +81,120 @@ class TestHttp(unittest.TestCase):
|
|||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, body: bytes, headers: dict[str, str] | None = None):
|
||||
self.body, self.headers = body, headers or {}
|
||||
|
||||
def read(self) -> bytes:
|
||||
return self.body
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
def http_error(code: int, headers: dict[str, str] | None = None):
|
||||
import email.message
|
||||
import urllib.error
|
||||
msg = email.message.Message()
|
||||
for k, v in (headers or {}).items():
|
||||
msg[k] = v
|
||||
return urllib.error.HTTPError("https://x.test", code, "err", msg, None)
|
||||
|
||||
|
||||
class TestFetchBytes(unittest.TestCase):
|
||||
def run_with(self, responses, **kw):
|
||||
from unittest import mock
|
||||
|
||||
from web_fetcher import core
|
||||
calls, sleeps = [], []
|
||||
|
||||
def opener(req, timeout):
|
||||
calls.append(req.full_url)
|
||||
r = responses.pop(0)
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return r
|
||||
with mock.patch.object(core, "_urlopen", opener), mock.patch.object(core, "_sleep", sleeps.append):
|
||||
try:
|
||||
return core.fetch_bytes("https://x.test/a", **kw), calls, sleeps
|
||||
except core.FetchError as exc:
|
||||
return exc, calls, sleeps
|
||||
|
||||
def test_404_fails_fast(self):
|
||||
res, calls, sleeps = self.run_with([http_error(404)])
|
||||
self.assertIn("HTTP 404", str(res))
|
||||
self.assertEqual((len(calls), sleeps), (1, []))
|
||||
|
||||
def test_503_honours_retry_after(self):
|
||||
res, calls, sleeps = self.run_with([http_error(503, {"Retry-After": "7"}), FakeResponse(b"ok")])
|
||||
self.assertEqual((res.body, len(calls), sleeps), (b"ok", 2, [7.0]))
|
||||
|
||||
def test_connection_errors_back_off_exponentially(self):
|
||||
import urllib.error
|
||||
res, calls, sleeps = self.run_with([urllib.error.URLError("down")] * 3)
|
||||
self.assertEqual((len(calls), sleeps), (3, [1.5, 3.0]))
|
||||
self.assertIn("down", str(res))
|
||||
|
||||
def test_last_modified_becomes_published(self):
|
||||
res, _, _ = self.run_with([FakeResponse(b"{}", {"Last-Modified": "Wed, 01 Oct 2026 08:00:00 GMT"})])
|
||||
self.assertEqual(res.published, "2026-10-01T08:00:00+00:00")
|
||||
|
||||
|
||||
class TestReplay(unittest.TestCase):
|
||||
def test_replay_reproduces_cached_run(self):
|
||||
import tempfile
|
||||
from unittest import mock
|
||||
|
||||
from web_fetcher import core
|
||||
body = json.dumps({"v": 3.2}).encode()
|
||||
spec = {"id": "r", "adapter": "http_json", "url": "https://x.test/r", "path": "v", "source": "test"}
|
||||
with tempfile.TemporaryDirectory() as cache:
|
||||
with mock.patch.object(core, "_urlopen", lambda req, timeout: FakeResponse(body)):
|
||||
first = run_spec(spec, cache_dir=cache)
|
||||
|
||||
def no_network(req, timeout):
|
||||
raise AssertionError("network used during replay")
|
||||
with mock.patch.object(core, "_urlopen", no_network):
|
||||
again = run_spec(spec, replay_dir=cache)
|
||||
missing = run_spec({**spec, "url": "https://x.test/other"}, replay_dir=cache)
|
||||
self.assertEqual([o.to_dict() for o in again], [o.to_dict() for o in first])
|
||||
self.assertEqual(first[0].value, 3.2)
|
||||
self.assertEqual(missing[0].status, "error")
|
||||
self.assertIn("not in replay cache", missing[0].note)
|
||||
|
||||
|
||||
class TestProvenanceRules(unittest.TestCase):
|
||||
def test_value_without_source_is_error(self):
|
||||
obs = run_spec({"id": "s", "adapter": "http_json", "url": "https://x.test", "path": "v"},
|
||||
get=fake(b'{"v": 1}'))
|
||||
self.assertEqual(obs[0].status, "error")
|
||||
self.assertIn("source", obs[0].note)
|
||||
|
||||
def csv(self, body: bytes, **extra):
|
||||
spec = {"id": "c", "adapter": "http_csv", "url": "https://x.test", "value_column": "value",
|
||||
"period_column": "period", "source": "test", **extra}
|
||||
return run_spec(spec, get=fake(body))[0]
|
||||
|
||||
def test_quarters_ordered_by_value_not_string(self):
|
||||
o = self.csv(b"period,value\n2025-Q4,1\n2026Q1,2\n")
|
||||
self.assertEqual((o.status, o.period, o.value), ("ok", "2026Q1", 2.0))
|
||||
self.assertEqual(o.note, "latest of 2 periods by quarter order")
|
||||
|
||||
def test_mixed_period_formats_are_error(self):
|
||||
o = self.csv(b"period,value\n2026-Q1,1\n2026-02,2\n")
|
||||
self.assertEqual(o.status, "error")
|
||||
self.assertIn("mixed period formats", o.note)
|
||||
|
||||
def test_explicit_string_order_is_allowed(self):
|
||||
o = self.csv(b"period,value\nb,1\na,2\n", period_format="string")
|
||||
self.assertEqual((o.status, o.period), ("ok", "b"))
|
||||
|
||||
def test_non_numeric_value_is_not_found_unless_string(self):
|
||||
body = b"period,value\n2026-01,(D)\n"
|
||||
self.assertEqual(self.csv(body).status, "not_found")
|
||||
self.assertEqual(self.csv(body, value_type="string").value, "(D)")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue