From 252195574949c9f394f6d4f77afa4f7ec2cdf277 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 19 Oct 2026 14:47:05 +0200 Subject: [PATCH] Pin checkpoint runs to web-fetcher v0.2.0; add --cache/--replay run_checkpoint.py refuses web-fetcher versions other than 0.2.0, records the version in record and summary, and passes --cache/--replay through to run_spec. Verified: live run with --cache and offline --replay give identical observations. Execution declaration, AGENTS.md and ASSEMBLY-WP-0014 updated. Requested by web-fetcher (WEBFETCH-WP-0003-T03). Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 193986@bnt-lap001 Assistant-Session: 585bfbea-320e-47b3-8c0e-bc1513a53307 --- AGENTS.md | 3 +- checkpoints/EXECUTION_DECLARATION.md | 4 +-- checkpoints/run_checkpoint.py | 35 ++++++++++++++----- checkpoints/test_run_checkpoint.py | 22 ++++++++++++ ...MBLY-WP-0014-checkpoint-operations-2027.md | 7 ++-- 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9d7d40f..fafa4db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,8 @@ run (writes files; delete them afterwards unless it is a real checkpoint): ```bash python3 -B -m unittest discover -s checkpoints -p 'test_*.py' -WEB_FETCHER_PATH=~/web-fetcher python3 -B checkpoints/run_checkpoint.py --kind light --date YYYY-MM-DD --no-llm +WEB_FETCHER_PATH=~/web-fetcher python3 -B checkpoints/run_checkpoint.py --kind light --date YYYY-MM-DD --no-llm [--cache DIR | --replay DIR] +# web-fetcher is pinned to v0.2.0 (checkout of that tag); other versions are refused ``` Use `python3 -B` after quick edits; a stale bytecode cache once masked a change. diff --git a/checkpoints/EXECUTION_DECLARATION.md b/checkpoints/EXECUTION_DECLARATION.md index aa88d9f..c38523f 100644 --- a/checkpoints/EXECUTION_DECLARATION.md +++ b/checkpoints/EXECUTION_DECLARATION.md @@ -34,9 +34,9 @@ profile. Decision for rein-aharness and glas-harness: see "Open questions". | Field | Value | |---|---| | Input schema | `--kind light\|full`, `--date YYYY-MM-DD` (Europe/Berlin day; default today). No other inputs. | -| Repository input | `coulomb/assembly-economics` at a verified Forgejo commit (main); `coulomb/web-fetcher` at a verified Forgejo commit (main). | +| Repository input | `coulomb/assembly-economics` at a verified Forgejo commit (main); `coulomb/web-fetcher` at tag `v0.2.0` (pinned 2026-10-19; the runner refuses any other version, `WEB_FETCHER_VERSION` in `run_checkpoint.py`). | | Code executed | `checkpoints/run_checkpoint.py` (standard library) and `web_fetcher` (standard library). No package installation. | -| Allowed write paths | `checkpoints//-.md`, `checkpoints/data/-.jsonl`. Nothing outside `checkpoints/`. | +| Allowed write paths | `checkpoints//-.md`, `checkpoints/data/-.jsonl`. Nothing outside `checkpoints/`. Optional `--cache DIR` (fetched bytes for offline `--replay`) must point to executor-local scratch outside the repository; it is not committed. | | Network egress | Hosts in `checkpoints/sources.json` (www.census.gov, ec.europa.eu, fred.stlouisfed.org, ghoapi.azureedge.net, api.worldbank.org, api.openalex.org) and the llm-connect endpoint. Nothing else. | | Model route | llm-connect, one request per run, prompt built only from fetched observations. Optional: `--no-llm` produces a valid record without it. | | Credential route | None for data sources (all public). llm-connect access per its own route (`warden route find "llm-connect inference"`). Forgejo push credential per the publish lane (below). No secrets in the repository or the record. | diff --git a/checkpoints/run_checkpoint.py b/checkpoints/run_checkpoint.py index a7bdf0b..73aabc3 100644 --- a/checkpoints/run_checkpoint.py +++ b/checkpoints/run_checkpoint.py @@ -11,7 +11,11 @@ Usage: python3 checkpoints/run_checkpoint.py --kind light [--date YYYY-MM-DD] [--no-llm] -Requires web-fetcher importable (pip install, or WEB_FETCHER_PATH=/path/to/web-fetcher). + [--cache DIR | --replay DIR] +Requires web-fetcher WEB_FETCHER_VERSION importable: pip install +"git+https://forgejo.coulomb.social/coulomb/web-fetcher@v0.2.0", or +WEB_FETCHER_PATH=/path/to/a checkout of that tag. Other versions are refused. +--cache stores the exact fetched bytes per run; --replay re-runs offline from them. Exit codes: 0 ok, 3 out of window, 1 failure. """ @@ -32,6 +36,8 @@ REPO = HERE.parent if os.environ.get("WEB_FETCHER_PATH"): sys.path.insert(0, os.environ["WEB_FETCHER_PATH"]) +WEB_FETCHER_VERSION = "0.2.0" # pinned tag v0.2.0; change only after re-verifying sources.json + WINDOW_RE = re.compile(r"(\d{4}-\d{2}-\d{2})\s*[–-]\s*(\d{4}-\d{2}-\d{2})|from\s+(\d{4}-\d{2}-\d{2})") @@ -56,7 +62,16 @@ def active_phase(day: date, kind: str, schedule: str) -> str | None: return None -def fetch(kind: str) -> list[dict]: +def web_fetcher_version() -> str: + import web_fetcher + + version = getattr(web_fetcher, "__version__", "unknown") + if version != WEB_FETCHER_VERSION: + raise SystemExit(f"web-fetcher {version} found; checkpoints are pinned to {WEB_FETCHER_VERSION}") + return version + + +def fetch(kind: str, cache_dir: str | None = None, replay_dir: str | None = None) -> list[dict]: from web_fetcher.registry import run_spec specs = json.loads((HERE / "sources.json").read_text())["specs"] @@ -64,7 +79,7 @@ def fetch(kind: str) -> list[dict]: for spec in specs: if kind == "light" and spec.get("kind") != "light": continue - for obs in run_spec(spec): + for obs in run_spec(spec, cache_dir=cache_dir, replay_dir=replay_dir): d = obs.to_dict() d.update(indicator=spec.get("indicator"), jurisdiction=spec.get("jurisdiction"), label=spec.get("label"), source=d.get("source") or spec.get("source")) @@ -134,7 +149,7 @@ def band_position(preds: dict, indicator: str, jur: str) -> str: def render(day: date, kind: str, phase: str, obs: list[dict], assess: str | None, assess_note: str | None, - data_rel: str) -> str: + data_rel: str, fetcher: str = WEB_FETCHER_VERSION) -> str: rows, missing = [], [] preds = registered() for o in obs: @@ -149,7 +164,7 @@ def render(day: date, kind: str, phase: str, obs: list[dict], assess: str | None f" — {o.get('note') or ''} Source: {o.get('source') or '—'}") lines = [f"# Checkpoint {day.isoformat()} ({kind})", "", f"- Model version: {model_version()}", f"- Phase: {phase}", - "- Executor: checkpoints/run_checkpoint.py (web-fetcher; llm-connect assessment)", + f"- Executor: checkpoints/run_checkpoint.py (web-fetcher {fetcher}; llm-connect assessment)", f"- Raw observations with provenance: `{data_rel}`", "", "## Observations", "", "| Indicator | Jurisdiction | Value | Unit | Reference period | Published | Source | Band position |", @@ -171,22 +186,26 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--kind", choices=["light", "full"], required=True) ap.add_argument("--date", default=None) ap.add_argument("--no-llm", action="store_true") + src = ap.add_mutually_exclusive_group() + src.add_argument("--cache", default=None, help="store fetched bytes here (replayable)") + src.add_argument("--replay", default=None, help="offline: replay a --cache directory") args = ap.parse_args(argv) day = date.fromisoformat(args.date) if args.date else berlin_today() phase = active_phase(day, args.kind, (HERE / "schedule.md").read_text()) if phase is None: print(json.dumps({"status": "out-of-window", "date": day.isoformat(), "kind": args.kind})) return 3 - obs = fetch(args.kind) + fetcher = web_fetcher_version() + obs = fetch(args.kind, cache_dir=args.cache, replay_dir=args.replay) data_rel = f"checkpoints/data/{day.isoformat()}-{args.kind}.jsonl" (REPO / data_rel).parent.mkdir(parents=True, exist_ok=True) (REPO / data_rel).write_text("\n".join(json.dumps(o, ensure_ascii=False, sort_keys=True) for o in obs) + "\n") assess, note = (None, "disabled (--no-llm)") if args.no_llm else assessment(obs) rec_rel = f"checkpoints/{day.year}/{day.isoformat()}-{args.kind}.md" (REPO / rec_rel).parent.mkdir(parents=True, exist_ok=True) - (REPO / rec_rel).write_text(render(day, args.kind, phase, obs, assess, note, data_rel)) + (REPO / rec_rel).write_text(render(day, args.kind, phase, obs, assess, note, data_rel, fetcher)) summary = {"status": "ok", "date": day.isoformat(), "kind": args.kind, "phase": phase, "path": rec_rel, - "data": data_rel, "ok": sum(o["status"] == "ok" for o in obs), + "data": data_rel, "web_fetcher": fetcher, "replay": bool(args.replay), "ok": sum(o["status"] == "ok" for o in obs), "not_found": sum(o["status"] == "not_found" for o in obs), "error": sum(o["status"] == "error" for o in obs), "assessment": bool(assess)} print(json.dumps(summary)) diff --git a/checkpoints/test_run_checkpoint.py b/checkpoints/test_run_checkpoint.py index 901adc0..8d7195b 100644 --- a/checkpoints/test_run_checkpoint.py +++ b/checkpoints/test_run_checkpoint.py @@ -2,6 +2,8 @@ python3 -B -m unittest discover -s checkpoints -p 'test_*.py' """ +import sys +import types import unittest from datetime import date from pathlib import Path @@ -48,5 +50,25 @@ class TestNumberGuard(unittest.TestCase): self.assertEqual(rc._numbers("EU at 19.95 % in 2025, US 23.8."), {"19.95", "2025", "23.8"}) +class TestWebFetcherPin(unittest.TestCase): + def _with_version(self, version): + saved = sys.modules.get("web_fetcher") + sys.modules["web_fetcher"] = types.SimpleNamespace(__version__=version) + try: + return rc.web_fetcher_version() + finally: + if saved is None: + del sys.modules["web_fetcher"] + else: + sys.modules["web_fetcher"] = saved + + def test_pinned_version_accepted(self): + self.assertEqual(self._with_version(rc.WEB_FETCHER_VERSION), rc.WEB_FETCHER_VERSION) + + def test_other_version_refused(self): + with self.assertRaises(SystemExit): + self._with_version("0.1.0") + + if __name__ == "__main__": unittest.main() diff --git a/workplans/ASSEMBLY-WP-0014-checkpoint-operations-2027.md b/workplans/ASSEMBLY-WP-0014-checkpoint-operations-2027.md index 09fd62b..f7796a8 100644 --- a/workplans/ASSEMBLY-WP-0014-checkpoint-operations-2027.md +++ b/workplans/ASSEMBLY-WP-0014-checkpoint-operations-2027.md @@ -10,7 +10,7 @@ depends_on: [ASSEMBLY-WP-0010] owner: claude-code topic_slug: coulomb-social created: "2026-10-13" -updated: "2026-10-14" +updated: "2026-10-19" state_hub_workstream_id: "e07ebc2e-2a59-5d7a-aebe-54d8622617a4" --- @@ -33,6 +33,8 @@ priority: high state_hub_task_id: "2d0f60c8-30c6-589f-ae7c-f26a01f03687" ``` +**2026-10-19:** web-fetcher pinned to tag `v0.2.0` (web-fetcher request, WEBFETCH-WP-0003-T03): the runner refuses other versions, records the version, and accepts `--cache DIR` / `--replay DIR`. Declaration updated. For the profile (step 2): the railiance01 runner should install the tag instead of a moving checkout and pass `--cache` to per-run scratch. + **2026-10-14, blocked on rein-aharness:** operator decisions 0636c577 (tenant-owned blueprint, no Glas profile) and c6f23433 (publisher owned by rein-aharness as a separately granted capability). Our side is done: owner declaration `activity-definitions/assembly-checkpoint.declaration.yaml` with the `repository_grant` v2 block, confirmed to glas-harness (GLAS-WP-0017-T02) and rein-aharness. Waiting for the publisher capability (REINAH-WP-0003-T04), then the canary run before 2026-12-15. **2026-10-13:** step 2 of rein-aharness's migration sequence prepared: @@ -54,8 +56,7 @@ Steps: (for example the Freedom Intelligence brief) already runs on one, so it can serve as the template. 2. Write the profile for `assembly-checkpoint`: executor host railiance01; - inputs (the assembly-economics commit, `~/web-fetcher` at its Forgejo - origin); outbound network only to the hosts listed in + inputs (the assembly-economics commit, web-fetcher at tag `v0.2.0`); outbound network only to the hosts listed in `checkpoints/sources.json` and llm-connect; write scope `checkpoints/`. 3. Replace the legacy publish route with a per-repo, path-scoped publish grant in rein-aharness's governed publish lane: `coulomb/assembly-economics`,