reuse-surface/tests/test_reports.py

440 lines
16 KiB
Python
Raw Normal View History

from __future__ import annotations
import argparse
import json
from pathlib import Path
REUSE-WP-0019-T05: reuse telemetry aggregation into R-axis evidence reuse_surface/reports.py: collect_reuse_events() merges the hub's GET /v1/reuse-events (if reachable) with this repo's local JSONL fallback, deduped. collect_reuse_report() aggregates per-capability consumer counts, outcome breakdown, and last-used. collect_reused_by_suggestions() proposes evidence-gated relation_add patches -- only for capabilities this repo owns, only for consumer repos not already listed -- reusing the existing patches.py:apply_patches mechanism (relation_add already isn't in SAFE_DETERMINISTIC_KINDS, so it was already never auto-applied by maintain --auto). New CLI: reuse-surface report reuse [--capability-id] [--format] [--suggest-relations] [--apply]. --apply requires --suggest-relations and is the only thing that writes -- nothing happens automatically from telemetry alone. schemas/capability.schema.yaml: added relations.reused_by as a new repoSlugList type, distinct from the existing capability-id relations, since reused-by targets are consumer repo slugs. specs/CapabilityMaturityStandard.md Sec8.9: what observed-reuse evidence counts toward R2->R3 (single corroborating consumer) vs R3->R4+ (multiple independent consumers) and what it never substitutes for. 19 new pytest cases, 162 total pass. Live-verified with synthetic local events against a real capability entry: --suggest-relations --apply correctly wrote relations.reused_by via the real apply_patches path (reverted after, since it was a smoke test). Deliberately deferred: surfacing consumer counts in the catalog/graph -- graph.py's relation model is capability-to-capability edges, a different namespace than repo-slug reused_by targets; catalog.py doesn't currently parse full front matter per entry. Left for a follow-up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 22:47:51 +02:00
import yaml
from reuse_surface.reports import (
cohort_filters_from_args,
collect_gap_report,
REUSE-WP-0019-T05: reuse telemetry aggregation into R-axis evidence reuse_surface/reports.py: collect_reuse_events() merges the hub's GET /v1/reuse-events (if reachable) with this repo's local JSONL fallback, deduped. collect_reuse_report() aggregates per-capability consumer counts, outcome breakdown, and last-used. collect_reused_by_suggestions() proposes evidence-gated relation_add patches -- only for capabilities this repo owns, only for consumer repos not already listed -- reusing the existing patches.py:apply_patches mechanism (relation_add already isn't in SAFE_DETERMINISTIC_KINDS, so it was already never auto-applied by maintain --auto). New CLI: reuse-surface report reuse [--capability-id] [--format] [--suggest-relations] [--apply]. --apply requires --suggest-relations and is the only thing that writes -- nothing happens automatically from telemetry alone. schemas/capability.schema.yaml: added relations.reused_by as a new repoSlugList type, distinct from the existing capability-id relations, since reused-by targets are consumer repo slugs. specs/CapabilityMaturityStandard.md Sec8.9: what observed-reuse evidence counts toward R2->R3 (single corroborating consumer) vs R3->R4+ (multiple independent consumers) and what it never substitutes for. 19 new pytest cases, 162 total pass. Live-verified with synthetic local events against a real capability entry: --suggest-relations --apply correctly wrote relations.reused_by via the real apply_patches path (reverted after, since it was a smoke test). Deliberately deferred: surfacing consumer counts in the catalog/graph -- graph.py's relation model is capability-to-capability edges, a different namespace than repo-slug reused_by targets; catalog.py doesn't currently parse full front matter per entry. Left for a follow-up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 22:47:51 +02:00
collect_reuse_events,
collect_reuse_report,
collect_reused_by_suggestions,
format_cohort_json,
format_cohort_markdown,
format_gap_markdown,
REUSE-WP-0019-T05: reuse telemetry aggregation into R-axis evidence reuse_surface/reports.py: collect_reuse_events() merges the hub's GET /v1/reuse-events (if reachable) with this repo's local JSONL fallback, deduped. collect_reuse_report() aggregates per-capability consumer counts, outcome breakdown, and last-used. collect_reused_by_suggestions() proposes evidence-gated relation_add patches -- only for capabilities this repo owns, only for consumer repos not already listed -- reusing the existing patches.py:apply_patches mechanism (relation_add already isn't in SAFE_DETERMINISTIC_KINDS, so it was already never auto-applied by maintain --auto). New CLI: reuse-surface report reuse [--capability-id] [--format] [--suggest-relations] [--apply]. --apply requires --suggest-relations and is the only thing that writes -- nothing happens automatically from telemetry alone. schemas/capability.schema.yaml: added relations.reused_by as a new repoSlugList type, distinct from the existing capability-id relations, since reused-by targets are consumer repo slugs. specs/CapabilityMaturityStandard.md Sec8.9: what observed-reuse evidence counts toward R2->R3 (single corroborating consumer) vs R3->R4+ (multiple independent consumers) and what it never substitutes for. 19 new pytest cases, 162 total pass. Live-verified with synthetic local events against a real capability entry: --suggest-relations --apply correctly wrote relations.reused_by via the real apply_patches path (reverted after, since it was a smoke test). Deliberately deferred: surfacing consumer counts in the catalog/graph -- graph.py's relation model is capability-to-capability edges, a different namespace than repo-slug reused_by targets; catalog.py doesn't currently parse full front matter per entry. Left for a follow-up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 22:47:51 +02:00
format_reuse_report_json,
format_reuse_report_markdown,
select_cohort,
)
SAMPLE_INDEX = {
"capabilities": [
{
"id": "capability.planning.only",
"vector": "D5 / A0 / C2 / R1",
"domain": "helix_forge",
"consumption_modes": ["planning"],
},
{
"id": "capability.implementation.ready",
"vector": "D5 / A4 / C3 / R3",
"domain": "helix_forge",
"consumption_modes": ["cli", "service API"],
},
{
"id": "capability.other.domain",
"vector": "D4 / A3 / C2 / R2",
"domain": "other",
"consumption_modes": ["cli"],
},
]
}
def test_planning_min_filter():
filters = cohort_filters_from_args(
argparse.Namespace(
planning_min="D5",
implementation_min=None,
discovery_min=None,
availability_min=None,
availability_max=None,
domain=None,
)
)
matches = select_cohort(filters, SAMPLE_INDEX)
assert [item["id"] for item in matches] == ["capability.planning.only"]
def test_implementation_min_filter():
filters = cohort_filters_from_args(
argparse.Namespace(
planning_min=None,
implementation_min="A4",
discovery_min=None,
availability_min=None,
availability_max=None,
domain=None,
)
)
matches = select_cohort(filters, SAMPLE_INDEX)
assert [item["id"] for item in matches] == ["capability.implementation.ready"]
def test_domain_filter():
filters = {"discovery_min": None, "availability_min": None, "availability_max": None, "domain": "helix_forge"}
matches = select_cohort(filters, SAMPLE_INDEX)
assert len(matches) == 2
def test_format_cohort_markdown_includes_filters():
filters = {"discovery_min": "D5", "availability_min": None, "availability_max": "A1", "domain": None}
text = format_cohort_markdown([SAMPLE_INDEX["capabilities"][0]], filters)
assert "planning-min" not in text
assert "discovery_min" in text
assert "capability.planning.only" in text
def test_format_cohort_json_payload():
filters = {"discovery_min": "D5", "availability_min": None, "availability_max": "A1", "domain": None}
payload = json.loads(
format_cohort_json([SAMPLE_INDEX["capabilities"][0]], filters)
)
assert payload["count"] == 1
assert payload["filters"]["discovery_min"] == "D5"
def test_collect_gap_report_from_roster():
root = Path(__file__).resolve().parent.parent
roster = root / "registry/federation/local-repo-roster.yaml"
report = collect_gap_report(roster)
assert report["summary"]["total"] == 62
assert len(report["publish_fail"]) == 0
assert report["unclassified_count"] + report["explicit_none_count"] == report["empty_scaffold_count"]
assert "/" in report["coverage_ratio"]
def test_collect_gap_report_splits_unclassified_and_explicit_none(tmp_path):
roster_path = tmp_path / "roster.yaml"
roster_path.write_text(
"""
summary:
total: 3
repos:
- slug: has-repo
status: established
capability_count: 1
capability_status: has
- slug: none-repo
status: established
capability_count: 0
capability_status: none
- slug: pending-repo
status: established
capability_count: 0
capability_status: pending
"""
)
report = collect_gap_report(roster_path, index={"capabilities": []})
assert report["unclassified"] == ["pending-repo"]
assert report["explicit_none"] == ["none-repo"]
assert report["empty_scaffold_count"] == 2
assert report["coverage_ratio"] == "2/3"
def test_format_gap_markdown_lists_publish_fail():
report = {
"roster_path": "/tmp/roster.yaml",
"summary": {"total": 60, "established": 60, "publish_pass": 48},
"publish_fail": [{"slug": "inter-hub", "publish_note": "missing repo"}],
"empty_scaffold_count": 1,
"empty_scaffolds": ["ops-bridge"],
"unclassified_count": 1,
"unclassified": ["ops-bridge"],
"explicit_none_count": 0,
"explicit_none": [],
"coverage_ratio": "59/60",
"seeded_repos": [],
"dedup_pending_local_owners": [],
"local_capability_count": 2,
}
text = format_gap_markdown(report)
assert "inter-hub" in text
assert "ops-bridge" in text
def test_cmd_report_gaps_json(monkeypatch):
from reuse_surface.cli import main
exit_code = main(["report", "gaps", "--format", "json"])
assert exit_code == 0
REUSE-WP-0018 T01/T02/T04/T06: plan-check deterministic matching + State Hub bridge T01: specs/PlanCheck.md design doc, plan-check-result.schema.json and reuse-event.schema.json (the latter shared with WP-0019's reuse telemetry). T02: reuse_surface/plan_check.py + 'reuse-surface plan-check' CLI command. Deterministic token-Jaccard matching against registry/indexes/federated.yaml (reuses overlaps.py's TOKEN_RE rather than a second scoring method), with reuse/extend/new verdicts, markdown and --format json output, staleness warning, and --record-outcome JSONL telemetry. T04: reuse_surface/statehub_bridge.py bridges plan-check 'new' verdicts to State Hub capability requests (--file-request) and surfaces open requests with no matching capability (report gaps --check-capability-requests, opt-in to stay offline-safe). Verified against the live local State Hub API; status field (not catalog_entry_id presence) is the correct open/closed signal, and the list endpoint needs a longer timeout than the health check (~7s observed with 5 rows). T06: docs (tools/README.md, RegistryFederation.md, SCOPE.md, IntentScopeGapAnalysis.md priority 29) and an informational CI smoke step. T03 (LLM rerank) not started -- llm-connect isn't running on this workstation. T05 (ecosystem rollout) remains blocked: WP-0017 has drafted entries for all 61 repos but they're still local-only pending its own T05 push/publish pass, so the federated index isn't yet worth rolling out plan-check as ecosystem convention. 16 new tests, all mocked -- no network calls in the default test run. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 00:57:18 +02:00
def test_cmd_report_gaps_check_capability_requests_unreachable(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr(
"reuse_surface.cli.list_open_capability_requests", lambda: None
)
exit_code = main(["report", "gaps", "--check-capability-requests"])
assert exit_code == 0
def test_cmd_report_gaps_check_capability_requests_json(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr(
"reuse_surface.cli.list_open_capability_requests",
lambda: [{"id": "r1", "title": "Need X", "requesting_domain_slug": "infotech"}],
)
exit_code = main(
["report", "gaps", "--check-capability-requests", "--format", "json"]
)
assert exit_code == 0
def test_cmd_report_cohorts_markdown(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr(
"reuse_surface.reports.load_index",
lambda: SAMPLE_INDEX,
)
exit_code = main(["report", "cohorts", "--planning-min", "D5"])
REUSE-WP-0019-T05: reuse telemetry aggregation into R-axis evidence reuse_surface/reports.py: collect_reuse_events() merges the hub's GET /v1/reuse-events (if reachable) with this repo's local JSONL fallback, deduped. collect_reuse_report() aggregates per-capability consumer counts, outcome breakdown, and last-used. collect_reused_by_suggestions() proposes evidence-gated relation_add patches -- only for capabilities this repo owns, only for consumer repos not already listed -- reusing the existing patches.py:apply_patches mechanism (relation_add already isn't in SAFE_DETERMINISTIC_KINDS, so it was already never auto-applied by maintain --auto). New CLI: reuse-surface report reuse [--capability-id] [--format] [--suggest-relations] [--apply]. --apply requires --suggest-relations and is the only thing that writes -- nothing happens automatically from telemetry alone. schemas/capability.schema.yaml: added relations.reused_by as a new repoSlugList type, distinct from the existing capability-id relations, since reused-by targets are consumer repo slugs. specs/CapabilityMaturityStandard.md Sec8.9: what observed-reuse evidence counts toward R2->R3 (single corroborating consumer) vs R3->R4+ (multiple independent consumers) and what it never substitutes for. 19 new pytest cases, 162 total pass. Live-verified with synthetic local events against a real capability entry: --suggest-relations --apply correctly wrote relations.reused_by via the real apply_patches path (reverted after, since it was a smoke test). Deliberately deferred: surfacing consumer counts in the catalog/graph -- graph.py's relation model is capability-to-capability edges, a different namespace than repo-slug reused_by targets; catalog.py doesn't currently parse full front matter per entry. Left for a follow-up. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 22:47:51 +02:00
assert exit_code == 0
# --- T05: reuse telemetry aggregation ---
SAMPLE_REUSE_EVENTS = [
{
"ts": "2026-07-01T00:00:00Z",
"consumer_repo": "repo-a",
"capability_id": "capability.infotech.issue-tracking",
"verdict": "reuse",
"outcome": "reused",
"source": "plan-check",
},
{
"ts": "2026-07-02T00:00:00Z",
"consumer_repo": "repo-b",
"capability_id": "capability.infotech.issue-tracking",
"verdict": "reuse",
"outcome": "reused",
"source": "manual",
},
{
"ts": "2026-07-03T00:00:00Z",
"consumer_repo": "repo-a",
"capability_id": "capability.audit.event-retain",
"verdict": "extend",
"outcome": "extended",
"source": "plan-check",
},
]
def test_collect_reuse_report_aggregates_per_capability():
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
assert report["total_events"] == 3
by_id = {row["capability_id"]: row for row in report["capabilities"]}
assert by_id["capability.infotech.issue-tracking"]["consumer_count"] == 2
assert by_id["capability.infotech.issue-tracking"]["consumer_repos"] == ["repo-a", "repo-b"]
assert by_id["capability.infotech.issue-tracking"]["outcomes"] == {"reused": 2}
assert by_id["capability.infotech.issue-tracking"]["last_used"] == "2026-07-02T00:00:00Z"
assert by_id["capability.audit.event-retain"]["consumer_count"] == 1
def test_collect_reuse_report_filters_by_capability_id():
report = collect_reuse_report(SAMPLE_REUSE_EVENTS, capability_id="capability.audit.event-retain")
assert len(report["capabilities"]) == 1
assert report["capabilities"][0]["capability_id"] == "capability.audit.event-retain"
def test_collect_reuse_report_ignores_events_without_capability_id():
events = SAMPLE_REUSE_EVENTS + [
{"ts": "2026-07-04T00:00:00Z", "consumer_repo": "repo-c", "capability_id": None, "verdict": "new", "outcome": "new", "source": "manual"}
]
report = collect_reuse_report(events)
assert report["total_events"] == 4
assert len(report["capabilities"]) == 2
def test_format_reuse_report_markdown_empty():
output = format_reuse_report_markdown({"capabilities": [], "total_events": 0})
assert "No reuse events recorded yet" in output
def test_format_reuse_report_markdown_nonempty():
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
output = format_reuse_report_markdown(report)
assert "capability.infotech.issue-tracking" in output
assert "3" in output
def test_format_reuse_report_json_roundtrips():
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
payload = json.loads(format_reuse_report_json(report))
assert payload["total_events"] == 3
def test_collect_reuse_events_merges_hub_and_local(tmp_path, monkeypatch):
monkeypatch.setattr(
"reuse_surface.hub_client.hub_list_reuse_events",
lambda capability_id=None, base_url=None: (200, {"events": [SAMPLE_REUSE_EVENTS[0]]}),
)
telemetry_dir = tmp_path / "registry" / "telemetry"
telemetry_dir.mkdir(parents=True)
(telemetry_dir / "plan-check-events.jsonl").write_text(
json.dumps(SAMPLE_REUSE_EVENTS[1], sort_keys=True) + "\n", encoding="utf-8"
)
events = collect_reuse_events(repo_root=tmp_path)
assert len(events) == 2
def test_collect_reuse_events_dedupes_identical_facts(tmp_path, monkeypatch):
monkeypatch.setattr(
"reuse_surface.hub_client.hub_list_reuse_events",
lambda capability_id=None, base_url=None: (200, {"events": [SAMPLE_REUSE_EVENTS[0]]}),
)
telemetry_dir = tmp_path / "registry" / "telemetry"
telemetry_dir.mkdir(parents=True)
(telemetry_dir / "plan-check-events.jsonl").write_text(
json.dumps(SAMPLE_REUSE_EVENTS[0], sort_keys=True) + "\n", encoding="utf-8"
)
events = collect_reuse_events(repo_root=tmp_path)
assert len(events) == 1
def test_collect_reuse_events_degrades_gracefully_when_hub_unreachable(tmp_path, monkeypatch):
monkeypatch.delenv("REUSE_SURFACE_URL", raising=False)
events = collect_reuse_events(repo_root=tmp_path)
assert events == []
def _write_entry_with_relations(tmp_path: Path, cap_id: str, reused_by: list[str]) -> str:
rel = "registry/capabilities/capability-demo-sample.md"
front_matter = {
"id": cap_id,
"name": "Sample",
"summary": "Sample",
"owner": "demo",
"status": "draft",
"domain": "helix_forge",
"tags": ["demo"],
"maturity": {
"discovery": {"current": "D3", "target": "D5", "confidence": "low"},
"availability": {"current": "A3", "target": "A4", "confidence": "low"},
},
"external_evidence": {
"completeness": {"level": "C2", "confidence": "low"},
"reliability": {"level": "R2", "confidence": "low"},
},
"discovery": {"intent": "demo", "includes": [], "excludes": []},
"availability": {
"current_level": "A3",
"target_level": "A4",
"current_artifacts": [],
"consumption_modes": ["informational"],
},
"relations": {"depends_on": [], "supports": [], "related_to": [], "reused_by": reused_by},
"evidence": {"documentation": [], "tests": []},
"consumer_guidance": {
"recommended_for": [],
"not_recommended_for": [],
"known_limitations": [],
},
}
path = tmp_path / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("---\n" + yaml.safe_dump(front_matter, sort_keys=False) + "---\n", encoding="utf-8")
return rel
def test_collect_reused_by_suggestions_skips_already_listed_consumers(tmp_path):
rel = _write_entry_with_relations(tmp_path, "capability.infotech.issue-tracking", reused_by=["repo-a"])
index_path = tmp_path / "registry" / "indexes" / "capabilities.yaml"
index_path.parent.mkdir(parents=True, exist_ok=True)
index_path.write_text(
yaml.safe_dump({
"version": 1, "domain": "helix_forge", "updated": "2026-07-01",
"capabilities": [{"id": "capability.infotech.issue-tracking", "path": rel, "vector": "D3/A3/C2/R2"}],
}),
encoding="utf-8",
)
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
suggestions = collect_reused_by_suggestions(report, repo_root=tmp_path)
# repo-a already listed -> only repo-b should be suggested
assert len(suggestions) == 1
assert suggestions[0]["value"] == {"type": "reused_by", "target": "repo-b"}
assert suggestions[0]["kind"] == "relation_add"
def test_collect_reused_by_suggestions_skips_capabilities_not_owned_locally(tmp_path):
# empty local index -- no capabilities owned here at all
index_path = tmp_path / "registry" / "indexes" / "capabilities.yaml"
index_path.parent.mkdir(parents=True, exist_ok=True)
index_path.write_text(
yaml.safe_dump({"version": 1, "domain": "helix_forge", "updated": "2026-07-01", "capabilities": []}),
encoding="utf-8",
)
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
suggestions = collect_reused_by_suggestions(report, repo_root=tmp_path)
assert suggestions == []
def test_collect_reused_by_suggestions_no_index_returns_empty(tmp_path):
report = collect_reuse_report(SAMPLE_REUSE_EVENTS)
suggestions = collect_reused_by_suggestions(report, repo_root=tmp_path)
assert suggestions == []
def test_cmd_report_reuse_markdown(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr("reuse_surface.cli.collect_reuse_events", lambda hub_url=None: SAMPLE_REUSE_EVENTS)
exit_code = main(["report", "reuse"])
assert exit_code == 0
def test_cmd_report_reuse_json(monkeypatch, capsys):
from reuse_surface.cli import main
monkeypatch.setattr("reuse_surface.cli.collect_reuse_events", lambda hub_url=None: SAMPLE_REUSE_EVENTS)
exit_code = main(["report", "reuse", "--format", "json"])
assert exit_code == 0
payload = json.loads(capsys.readouterr().out)
assert payload["total_events"] == 3
def test_cmd_report_reuse_apply_without_suggest_relations_errors(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr("reuse_surface.cli.collect_reuse_events", lambda hub_url=None: [])
exit_code = main(["report", "reuse", "--apply"])
assert exit_code == 1
def test_cmd_report_reuse_suggest_relations(monkeypatch, capsys):
from reuse_surface.cli import main
monkeypatch.setattr("reuse_surface.cli.collect_reuse_events", lambda hub_url=None: SAMPLE_REUSE_EVENTS)
monkeypatch.setattr(
"reuse_surface.cli.collect_reused_by_suggestions",
lambda report: [{"capability_id": "capability.infotech.issue-tracking", "kind": "relation_add", "detail": "observed reuse by repo-a", "value": {"type": "reused_by", "target": "repo-a"}}],
)
exit_code = main(["report", "reuse", "--suggest-relations"])
assert exit_code == 0
out = capsys.readouterr().out
assert "repo-a" in out
def test_cmd_report_reuse_apply_calls_apply_patches(monkeypatch):
from reuse_surface.cli import main
monkeypatch.setattr("reuse_surface.cli.collect_reuse_events", lambda hub_url=None: SAMPLE_REUSE_EVENTS)
suggestion = {"capability_id": "capability.infotech.issue-tracking", "kind": "relation_add", "detail": "x", "value": {"type": "reused_by", "target": "repo-a"}}
monkeypatch.setattr("reuse_surface.cli.collect_reused_by_suggestions", lambda report: [suggestion])
applied_calls = []
monkeypatch.setattr(
"reuse_surface.patches.apply_patches",
lambda repo_root, patches: applied_calls.append(patches) or ["applied one"],
)
exit_code = main(["report", "reuse", "--suggest-relations", "--apply"])
assert exit_code == 0
assert applied_calls == [[suggestion]]