From fd0d0d537b16e10a7053dba98b638e4eeca2a507 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 25 Aug 2026 23:34:25 +0200 Subject: [PATCH] feat(projection): derive a repository's projection from the forge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements ADR-012 decisions 1 and 2 (STATE-WP-0083 T01, T02 partial). Central clones the default branch from Forgejo and derives its own projection: 69 workplans and 459 tasks from the-custodian at d5013ae, identical across runs, with the commit recorded as provenance. Identifiers are derived in the ADR-007 namespace and verified against live records, so a forge-derived projection and a preliminary overlay agree on identity without reconciliation. The diff first matched hub records by UUID and was badly wrong: most hub records carry pre-ADR-007 random identifiers, so nearly everything appeared simultaneously missing and stale, and a reset built on it would have destroyed and recreated the entire projection. It now matches canonical record id, falling back to the backing file. whitehat-security — bootstrapped straight from files — now reports clean, which is the control. Task-level comparison is deliberately not trusted: hub tasks carry no canonical record id, only a title, so matching is by title. Recorded as T06; T03 is limited to workplans until it lands. Co-Authored-By: Claude Opus 5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 2583210@bnt-lap001 Assistant-Session: f2bff2d5-e9b2-4338-92ca-10282a927006 --- api/services/forge_projection.py | 380 ++++++++++++++++++ tests/test_forge_projection.py | 98 +++++ ...-WP-0083-forge-derived-projection-reset.md | 74 +++- 3 files changed, 550 insertions(+), 2 deletions(-) create mode 100644 api/services/forge_projection.py create mode 100644 tests/test_forge_projection.py diff --git a/api/services/forge_projection.py b/api/services/forge_projection.py new file mode 100644 index 0000000..ee8b1e7 --- /dev/null +++ b/api/services/forge_projection.py @@ -0,0 +1,380 @@ +"""Derive a repository's work-record projection from the forge (STATE-WP-0083-T01). + +`ADR-012` decision 1 makes the forge the projection source. Central does its own +reading: it clones the repository's default branch and derives from that, rather +than accepting a projection computed elsewhere — which `ADR-010` decision 5 +forbids. + +This module is read-only. Deriving must be safe to run at any time, because it is +what makes the reset in `T03` verifiable: you can always ask what the projection +*should* be without changing anything. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +# Same derivation as ADR-007 / repo-manager, so a forge-derived projection and a +# preliminary overlay compute identical identities for the same record. +_WORK_RECORD_NAMESPACE = uuid.UUID("a4058507-5c4a-5a00-ab06-fffa4fb46009") +_TASK_BLOCK_RE = re.compile(r"```task\s*\n(.*?)\n```", re.DOTALL) +_HEADING_RE = re.compile(r"^(#{1,4})\s+(.+?)$", re.MULTILINE) +DEFAULT_FORGE_BASE = "https://forgejo.coulomb.social/coulomb" + + +def derived_record_uuid(record_id: str) -> str: + return str(uuid.uuid5(_WORK_RECORD_NAMESPACE, f"helixforge\n{record_id}")) + + +class ForgeDeriveError(RuntimeError): + """The repository could not be read from the forge.""" + + +@dataclass +class DerivedTask: + record_id: str + uuid: str + title: str | None + status: str | None + priority: str | None + + +@dataclass +class DerivedWorkplan: + record_id: str + uuid: str + title: str | None + status: str | None + relative_path: str + archived: bool + tasks: list[DerivedTask] = field(default_factory=list) + + +@dataclass +class DerivedProjection: + repo_slug: str + commit: str + workplans: list[DerivedWorkplan] = field(default_factory=list) + + @property + def task_count(self) -> int: + return sum(len(w.tasks) for w in self.workplans) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.forge-projection.v1", + "repo_slug": self.repo_slug, + # Provenance is not optional: a projection that cannot name the + # commit it came from cannot be audited (ADR-012 decision 2). + "commit": self.commit, + "workplans": [ + { + "record_id": w.record_id, + "uuid": w.uuid, + "title": w.title, + "status": w.status, + "relative_path": w.relative_path, + "archived": w.archived, + "tasks": [ + { + "record_id": t.record_id, + "uuid": t.uuid, + "title": t.title, + "status": t.status, + "priority": t.priority, + } + for t in w.tasks + ], + } + for w in self.workplans + ], + } + + +def _run_git(*args: str, cwd: str | None = None, timeout: float = 120.0) -> str: + proc = subprocess.run( + ["git", *args], cwd=cwd, capture_output=True, text=True, timeout=timeout + ) + if proc.returncode != 0: + raise ForgeDeriveError((proc.stderr or proc.stdout).strip()[:400]) + return proc.stdout.strip() + + +def _split_frontmatter(text: str) -> tuple[dict, str]: + if not text.startswith("---"): + return {}, text + end = text.find("\n---", 3) + if end == -1: + return {}, text + raw = text[3:end] + body = text[end + 4 :] + try: + meta = yaml.safe_load(raw) or {} + except yaml.YAMLError: + return {}, text + return (meta if isinstance(meta, dict) else {}), body + + +def _parse_tasks(body: str) -> list[DerivedTask]: + headings = [ + (m.start(), m.group(2).strip()) for m in _HEADING_RE.finditer(body) + ] + out: list[DerivedTask] = [] + for m in _TASK_BLOCK_RE.finditer(body): + try: + block = yaml.safe_load(m.group(1).strip()) or {} + except yaml.YAMLError: + continue + if not isinstance(block, dict): + continue + rid = str(block.get("id") or "").strip() + if not rid: + continue + title = block.get("title") + if not title: + prev = [t for pos, t in headings if pos < m.start()] + title = prev[-1] if prev else None + out.append( + DerivedTask( + record_id=rid, + # Derived, not read from the file: the forge projection must not + # inherit an identifier the file happens to carry. + uuid=derived_record_uuid(rid), + title=title, + status=(str(block["status"]).strip() if block.get("status") else None), + priority=(str(block["priority"]).strip() if block.get("priority") else None), + ) + ) + return out + + +def derive_from_checkout(repo_root: Path, repo_slug: str, commit: str) -> DerivedProjection: + """Derive a projection from an already-materialised checkout.""" + proj = DerivedProjection(repo_slug=repo_slug, commit=commit) + wp_dir = repo_root / "workplans" + if not wp_dir.is_dir(): + return proj + for path in sorted(wp_dir.rglob("*.md")): + if path.name.startswith("."): + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + meta, body = _split_frontmatter(text) + if str(meta.get("type") or "").strip() != "workplan": + continue + rid = str(meta.get("id") or "").strip() + if not rid: + continue + proj.workplans.append( + DerivedWorkplan( + record_id=rid, + uuid=derived_record_uuid(rid), + title=(str(meta["title"]).strip() if meta.get("title") else None), + status=(str(meta["status"]).strip() if meta.get("status") else None), + relative_path=str(path.relative_to(repo_root).as_posix()), + archived=path.parent.name == "archived", + tasks=_parse_tasks(body), + ) + ) + proj.workplans.sort(key=lambda w: w.record_id) + return proj + + +def derive_from_forge( + repo_slug: str, *, forge_base: str = DEFAULT_FORGE_BASE, ref: str | None = None +) -> DerivedProjection: + """Clone the repository's default branch from the forge and derive from it. + + A fresh shallow clone every time, deliberately: the source is what the forge + holds now, and a reused working copy is how a projection ends up reflecting + someone's local state instead (ADR-012 context). + """ + url = f"{forge_base.rstrip('/')}/{repo_slug}.git" + with tempfile.TemporaryDirectory(prefix=f"forge-{repo_slug}-") as tmp: + args = ["clone", "--depth", "1", "--quiet"] + if ref: + args += ["--branch", ref] + try: + _run_git(*args, url, tmp) + except subprocess.TimeoutExpired as exc: + raise ForgeDeriveError(f"clone timed out for {repo_slug}") from exc + commit = _run_git("rev-parse", "HEAD", cwd=tmp) + return derive_from_checkout(Path(tmp), repo_slug, commit) + + +# --------------------------------------------------------------------------- +# Comparison against what the hub currently holds (STATE-WP-0083-T02) +# --------------------------------------------------------------------------- + + + +_ARCHIVE_PREFIX_RE = re.compile(r"^\d{6}-") + + +def _path_key(path: str) -> str: + """Normalise a workplan path so an archived copy matches its live one.""" + name = path.rsplit("/", 1)[-1] + return _ARCHIVE_PREFIX_RE.sub("", name).strip().lower() + + +@dataclass +class ProjectionDiff: + """What a reset would change, computed without changing anything. + + This is what makes `ADR-012` decision 7's "verifiable" real: the reset can + always be inspected before it runs, and its result compared against the forge + afterwards. + """ + + repo_slug: str + commit: str + missing: list[dict[str, Any]] = field(default_factory=list) # forge has, hub lacks + stale: list[dict[str, Any]] = field(default_factory=list) # hub has, forge lacks + differing: list[dict[str, Any]] = field(default_factory=list) # both, fields differ + + @property + def clean(self) -> bool: + return not (self.missing or self.stale or self.differing) + + @property + def would_remove(self) -> int: + return len(self.stale) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "state-hub.projection-diff.v1", + "repo_slug": self.repo_slug, + "commit": self.commit, + "clean": self.clean, + "counts": { + "missing": len(self.missing), + "stale": len(self.stale), + "differing": len(self.differing), + }, + "missing": self.missing, + "stale": self.stale, + "differing": self.differing, + } + + +def diff_against_hub( + derived: DerivedProjection, + hub_workplans: list[dict[str, Any]], + hub_tasks_by_workplan: dict[str, list[dict[str, Any]]], +) -> ProjectionDiff: + """Compare a derived projection with the hub's current records. + + Pure: takes the hub's state as data rather than reading it, so the comparison + is testable without a database and cannot accidentally mutate anything. + """ + d = ProjectionDiff(repo_slug=derived.repo_slug, commit=derived.commit) + + # Match on canonical identity, never on UUID. Most hub records still carry + # pre-ADR-007 random identifiers, so a UUID-keyed comparison reports every + # record as simultaneously missing and stale — and a reset built on that + # would destroy and recreate the entire projection. The canonical record id + # is what is stable across the identifier migration; the backing file is the + # fallback when a hub slug was derived from a filename rather than an id. + def _wp_key(record_id: str | None, slug: str | None, path: str | None) -> str: + if record_id: + return record_id.strip().lower() + if path: + return _path_key(path) + return (slug or "").strip().lower() + + want_wp = {w.record_id.strip().lower(): w for w in derived.workplans} + have_wp: dict[str, dict[str, Any]] = {} + want_paths = {_path_key(w.relative_path): k for k, w in want_wp.items()} + for w in hub_workplans: + slug = str(w.get("slug") or "") + key = slug.strip().lower() + if key not in want_wp: + # Slugs were not always the canonical id; fall back to the file. + bp = w.get("backing_relative_path") + if bp and _path_key(bp) in want_paths: + key = want_paths[_path_key(bp)] + else: + cand = [k for k in want_wp if slug.lower().startswith(k + "-")] + if len(cand) == 1: + key = cand[0] + have_wp[key] = w + + for key, w in want_wp.items(): + if key not in have_wp: + d.missing.append({"kind": "workplan", "record_id": w.record_id, "uuid": w.uuid}) + for key, w in have_wp.items(): + if key not in want_wp: + uid = str(w["id"]) + d.stale.append( + { + "kind": "workplan", + "uuid": uid, + "slug": w.get("slug"), + "status": w.get("status"), + # A stale record with no backing file is the case that must + # never be destroyed silently (ADR-012 decision 7). + "has_backing_file": bool( + w.get("backing_filename") or w.get("backing_relative_path") + ), + } + ) + for key, w in want_wp.items(): + cur = have_wp.get(key) + uid = str(cur["id"]) if cur else w.uuid + if not cur: + continue + changed = {} + if (cur.get("status") or None) != (w.status or None): + changed["status"] = {"hub": cur.get("status"), "forge": w.status} + cur_path = cur.get("backing_relative_path") or None + if cur_path != w.relative_path: + changed["backing_relative_path"] = {"hub": cur_path, "forge": w.relative_path} + if changed: + d.differing.append( + {"kind": "workplan", "record_id": w.record_id, "uuid": uid, "changed": changed} + ) + + for w in derived.workplans: + hub_rows = hub_tasks_by_workplan.get(w.uuid, []) + if not hub_rows: + cur = have_wp.get(w.record_id.strip().lower()) + if cur: + hub_rows = hub_tasks_by_workplan.get(str(cur["id"]), []) + # Tasks carry no canonical id on the hub, only a title, so compare on + # title. Imperfect, and the reason task removal needs the same explicit + # acknowledgement as everything else. + have = {(t.get("title") or "").strip().lower(): t for t in hub_rows} + want = {(t.title or t.record_id).strip().lower(): t for t in w.tasks} + for key, t in want.items(): + if key not in have: + d.missing.append( + {"kind": "task", "record_id": t.record_id, "uuid": t.uuid, + "workplan": w.record_id} + ) + for key, t in have.items(): + if key not in want: + uid = str(t["id"]) + d.stale.append( + {"kind": "task", "uuid": uid, "title": t.get("title"), + "status": t.get("status"), "workplan": w.record_id, + "has_backing_file": False} + ) + for key, t in want.items(): + cur = have.get(key) + if cur and (cur.get("status") or None) != (t.status or None): + d.differing.append( + {"kind": "task", "record_id": t.record_id, "uuid": t.uuid, + "workplan": w.record_id, + "changed": {"status": {"hub": cur.get("status"), "forge": t.status}}} + ) + return d diff --git a/tests/test_forge_projection.py b/tests/test_forge_projection.py new file mode 100644 index 0000000..8268ea7 --- /dev/null +++ b/tests/test_forge_projection.py @@ -0,0 +1,98 @@ +"""Deriving a projection from the forge (STATE-WP-0083-T01). + +The properties that matter are identity and determinism: a forge-derived +projection must compute the same record identities as repo-manager, and the same +commit must always yield the same projection. Without both, the reset in T03 +cannot be verified against anything. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from api.services import forge_projection as fp + + +def test_identity_matches_the_fleet_derivation(): + """Same namespace as ADR-007, so overlay and forge agree on identity.""" + assert fp.derived_record_uuid("CUST-WP-0067") == "16249302-2767-55df-aec0-d92c2751c225" + assert fp.derived_record_uuid("CUST-WP-0067-T01") == "f3608db4-20a5-58fb-a965-885eb14858af" + + +def _repo(tmp_path: Path) -> Path: + root = tmp_path / "demo" + (root / "workplans" / "archived").mkdir(parents=True) + (root / "workplans" / "DEMO-WP-0001-a.md").write_text( + "---\nid: DEMO-WP-0001\ntype: workplan\ntitle: \"First\"\nstatus: active\n---\n\n" + "## Do the thing\n\n```task\nid: DEMO-WP-0001-T01\nstatus: todo\npriority: high\n```\n\n" + "## Do the other\n\n```task\nid: DEMO-WP-0001-T02\nstatus: done\npriority: low\n```\n", + encoding="utf-8", + ) + (root / "workplans" / "archived" / "260101-DEMO-WP-0002-b.md").write_text( + "---\nid: DEMO-WP-0002\ntype: workplan\ntitle: \"Second\"\nstatus: finished\n---\n\n# b\n", + encoding="utf-8", + ) + (root / "workplans" / "NOTES.md").write_text( + "---\nid: NOT-A-WORKPLAN\ntype: note\n---\n\n# not a workplan\n", encoding="utf-8" + ) + return root + + +def test_derives_workplans_tasks_and_archived_flag(tmp_path): + p = fp.derive_from_checkout(_repo(tmp_path), "demo", "abc123") + assert [w.record_id for w in p.workplans] == ["DEMO-WP-0001", "DEMO-WP-0002"] + first, second = p.workplans + assert first.status == "active" and first.archived is False + assert second.archived is True + assert p.task_count == 2 + assert [t.record_id for t in first.tasks] == ["DEMO-WP-0001-T01", "DEMO-WP-0001-T02"] + + +def test_ignores_files_that_are_not_workplans(tmp_path): + """Selection is by `type: workplan`; anything else is not this hub's business.""" + p = fp.derive_from_checkout(_repo(tmp_path), "demo", "abc123") + assert all(w.record_id != "NOT-A-WORKPLAN" for w in p.workplans) + + +def test_task_titles_fall_back_to_the_preceding_heading(tmp_path): + p = fp.derive_from_checkout(_repo(tmp_path), "demo", "abc123") + titles = [t.title for t in p.workplans[0].tasks] + assert titles == ["Do the thing", "Do the other"] + + +def test_identifiers_are_derived_not_read_from_the_file(tmp_path): + """A forge projection must not inherit whatever id a file happens to carry.""" + root = _repo(tmp_path) + f = root / "workplans" / "DEMO-WP-0001-a.md" + f.write_text( + f.read_text(encoding="utf-8").replace( + "status: active", + 'status: active\nstate_hub_workstream_id: "00000000-0000-4000-8000-000000000000"', + ), + encoding="utf-8", + ) + p = fp.derive_from_checkout(root, "demo", "abc123") + assert p.workplans[0].uuid == fp.derived_record_uuid("DEMO-WP-0001") + assert p.workplans[0].uuid != "00000000-0000-4000-8000-000000000000" + + +def test_same_input_yields_identical_output(tmp_path): + root = _repo(tmp_path) + assert fp.derive_from_checkout(root, "demo", "abc").to_dict() == \ + fp.derive_from_checkout(root, "demo", "abc").to_dict() + + +def test_missing_workplans_directory_is_empty_not_an_error(tmp_path): + (tmp_path / "bare").mkdir() + p = fp.derive_from_checkout(tmp_path / "bare", "bare", "abc") + assert p.workplans == [] and p.commit == "abc" + + +def test_clone_failure_is_reported_not_swallowed(monkeypatch): + def boom(*a, **k): + raise fp.ForgeDeriveError("repository not found") + monkeypatch.setattr(fp, "_run_git", boom) + with pytest.raises(fp.ForgeDeriveError, match="not found"): + fp.derive_from_forge("nope") diff --git a/workplans/STATE-WP-0083-forge-derived-projection-reset.md b/workplans/STATE-WP-0083-forge-derived-projection-reset.md index bd8c560..cc3b1ce 100644 --- a/workplans/STATE-WP-0083-forge-derived-projection-reset.md +++ b/workplans/STATE-WP-0083-forge-derived-projection-reset.md @@ -54,7 +54,7 @@ and should be extracted rather than rewritten. ```task id: STATE-WP-0083-T01 -status: todo +status: done priority: high ``` @@ -70,11 +70,22 @@ worthless precisely because nothing ever wrote it correctly. Acceptance: deriving `the-custodian` twice from the same commit yields identical output, and the commit is reported. +**Done (2026-08-25).** `api/services/forge_projection.py`. Central clones the +default branch from Forgejo and derives 69 workplans and 459 tasks from +`the-custodian` at commit `d5013ae`, identically across runs. Identifiers are +derived from the canonical record id in the `ADR-007` namespace, verified against +live records — so a forge-derived projection and a preliminary overlay agree on +identity without reconciliation. Eight tests, including that identifiers are +derived rather than inherited from whatever a file happens to carry. + +A fresh shallow clone each time is deliberate: reusing a working copy is how a +projection ends up reflecting someone's local state instead of the forge. + ## Report the difference before changing anything ```task id: STATE-WP-0083-T02 -status: todo +status: progress priority: high ``` @@ -90,6 +101,40 @@ clear — before anyone commits to clearing them. Acceptance: a dry-run diff for a repository with known drift matches what manual inspection shows. +**Workplan-level diff done; task-level blocked (2026-08-25).** + +The first implementation matched hub records to derived ones *by UUID* and was +badly wrong: most hub records still carry pre-`ADR-007` random identifiers, so +nearly every record appeared simultaneously missing and stale. A reset built on +that comparison would have destroyed and recreated the whole projection. It now +matches on canonical record id, falling back to the backing file. + +Workplan-level results are trustworthy and inspectable: + +| Repository | missing | stale | differing | +|---|---|---|---| +| `whitehat-security` | 0 | 0 | 0 | +| `the-custodian` | 3 | 4 | 52 | +| `railiance-platform` | 18 | 25 | 2 | + +`whitehat-security` reporting clean is the control: it was bootstrapped directly +from its files, so a forge derivation must agree with it exactly. The four stale +workplans on `the-custodian` were checked by hand and are real — +`CUST-WP-0023`/`0024` have no file in the repository at all, and +`state-hub-v0.1`/`v0.2` carry slugs that were never canonical ids. All four are +genuine hub-first records, the class `ADR-010` says to disposition. + +**Blocked: hub tasks carry no canonical record id.** The task schema is +`id, workplan_id, title, status, priority, …` with nothing holding +`CUST-WP-0067-T01`. A file task and a hub task can therefore only be matched by +title, which is why the task diff reports 149 missing and 131 stale for +`the-custodian` where the workplan diff reports 3 and 4. That is the matching +failing, not drift. + +Task-level reset must not be built on title matching — renaming a task heading +would silently destroy and recreate its record. `T03` is limited to workplans +until tasks carry their canonical id, which is `T06`. + ## Apply the reset transactionally ```task @@ -144,3 +189,28 @@ replacement belongs with the provenance work in `T01`. Acceptance: `CUST-WP-0068-T09` closes; no fingerprint input depends on a local filesystem. + + +## Give hub tasks their canonical record id + +```task +id: STATE-WP-0083-T06 +status: todo +priority: high +``` + +Hub task rows hold no canonical identifier — only a title — so nothing reliably +connects `CUST-WP-0067-T01` in a file to its row. Every other record type has a +stable identity; tasks do not, and that gap is what stops the reset from covering +them. + +Add the canonical id to the task record and populate it during derivation and +registration. Once present, task matching becomes identity-based like workplans, +and `T03` can extend to tasks safely. + +Until then a task's identity is its title, which changes whenever someone edits a +heading. That is not a foundation for deletion. + +Acceptance: task rows carry their canonical record id; the task diff for +`whitehat-security` reports clean; `the-custodian`'s task diff falls to something +manual inspection confirms.