coulomb/repo-scoping
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(consistency): sync task status from DB [auto]

Browse source 
Updated by fix-consistency on 2026-05-15:
  - update .custodian-brief.md for repo-scoping
This commit is contained in:
tegwick 2026-05-15 21:14:21 +02:00
parent f38ed6847c
commit 084159e51c
42 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/repo_scoping/self_scoping/__init__.py Normal file
View file

@ -0,0 +1,13 @@
from repo_registry.self_scoping.assessment import export_assessment_artifact
from repo_registry.self_scoping.comparison import compare_assessment_to_golden
from repo_registry.self_scoping.review_store import (
record_assessment_outcome,
record_assessment_pair_outcome,
)
__all__ = [
"compare_assessment_to_golden",
"export_assessment_artifact",
"record_assessment_outcome",
"record_assessment_pair_outcome",
]

478
src/repo_scoping/self_scoping/assessment.py Normal file
View file

@ -0,0 +1,478 @@
from __future__ import annotations
import json
import subprocess
from collections import Counter
from dataclasses import asdict
from datetime import UTC, datetime
from importlib import metadata
from pathlib import Path
from typing import Any
from repo_registry.acceptance import (
active_quality_criteria_version,
evaluate_candidate_graph_quality,
quality_gate_outcome_dicts,
)
from repo_registry.core.models import (
Ability,
CandidateAbility,
CandidateCapability,
CandidateEvidence,
CandidateFeature,
ContentChunk,
ObservedFact,
RepositoryAbilityMap,
ReviewDecision,
SourceReference,
)
from repo_registry.core.service import RegistryService
SCHEMA_VERSION = "self-scoping-assessment/v1"
KNOWN_PROVIDER_ROUTING_CAPABILITY = "Route LLM Requests Across Providers"
def export_assessment_artifact(
service: RegistryService,
repository_id: int,
analysis_run_id: int,
*,
role: str = "challenger",
outcome: str = "challenger",
reviewer: str = "codex",
summary: str | None = None,
engine_root: str | Path | None = None,
) -> dict[str, Any]:
"""Export a completed analysis run as a self-scoping assessment artifact."""
repository = service.get_repository(repository_id)
analysis_run = service.get_analysis_run(repository_id, analysis_run_id)
if analysis_run.status != "completed":
raise ValueError(
f"analysis run {analysis_run_id} is {analysis_run.status}, not completed"
)
snapshot = (
service.store.get_snapshot(analysis_run.snapshot_id)
if analysis_run.snapshot_id is not None
else None
)
facts = service.list_observed_facts(repository_id, analysis_run_id)
chunks = service.list_content_chunks(repository_id, analysis_run_id)
graph = service.candidate_graph(repository_id, analysis_run_id)
gate_outcomes = evaluate_candidate_graph_quality(graph)
ability_map = service.ability_map(repository_id)
decisions = service.list_review_decisions(repository_id, analysis_run_id)
engine_identity = _engine_identity(
analysis_run.scanner_version,
Path(engine_root or Path.cwd()),
)
regression_patterns = _known_regression_patterns(graph.abilities, decisions)
comparison_eligibility = _comparison_eligibility(
role,
engine_identity["release_binding_status"],
)
artifact_summary = summary or _summary(role, regression_patterns)
return {
"schema_version": SCHEMA_VERSION,
"artifact_id": _artifact_id(repository.name, analysis_run_id, role),
"artifact_type": "assessment_run",
"created_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"target_repository": {
"repo_slug": _slug(repository.name),
"repository_id": repository.id,
"source": snapshot.source_path if snapshot is not None else repository.url,
"target_commit": snapshot.commit_hash if snapshot is not None else "unknown",
"target_branch": snapshot.branch if snapshot is not None else repository.branch,
"dirty_state": _dirty_state(Path(snapshot.source_path)) if snapshot is not None else "unknown",
"file_count": snapshot.file_count if snapshot is not None else None,
},
"engine_identity": engine_identity,
"execution": {
"mode": _execution_mode(decisions),
"analysis_run_id": analysis_run.id,
"candidate_source": _candidate_source(decisions),
"acceptance_mode": _acceptance_mode(decisions),
"started_at": _timestamp(analysis_run.started_at),
"completed_at": _timestamp(analysis_run.completed_at),
},
"assessment": {
"role": role,
"outcome": outcome,
"summary": artifact_summary,
"reviewer": reviewer,
"comparison_eligibility": comparison_eligibility,
"rationale": _rationale(regression_patterns, comparison_eligibility),
},
"fact_summary": _fact_summary(facts),
"content_chunk_summary": _content_chunk_summary(chunks),
"generated_tree": {
"abilities": [_candidate_ability(ability) for ability in graph.abilities]
},
"approved_map": _approved_map(ability_map),
"review_decisions": [_review_decision(decision) for decision in decisions],
"quality_gate_outcomes": quality_gate_outcome_dicts(gate_outcomes),
"known_regression_patterns": regression_patterns,
"notes": [
"Generated by repo-scoping self-scoping assessment exporter.",
(
"Artifact is not comparable as a preferred baseline until engine "
"identity is complete."
if comparison_eligibility == "not_comparable"
else "Artifact has enough engine identity metadata for comparison."
),
],
}
def _engine_identity(scanner_version: str, engine_root: Path) -> dict[str, Any]:
engine_commit = _git_value(engine_root, "rev-parse", "HEAD")
dirty_state = _dirty_state(engine_root)
release = _git_value(engine_root, "describe", "--tags", "--exact-match")
release_binding_status = "complete" if engine_commit else "unbound"
return {
"repo_scoping_version": _package_version(),
"engine_commit": engine_commit,
"engine_release": release,
"engine_dirty_state": dirty_state,
"scanner_version": scanner_version,
"candidate_generator_version": "unversioned",
"quality_criteria_version": active_quality_criteria_version(),
"prompt_version": None,
"release_binding_status": release_binding_status,
"release_binding_note": (
"Engine commit was captured from git."
if engine_commit
else "Engine commit could not be captured; artifact is not comparable."
),
}
def _package_version() -> str:
try:
return metadata.version("repo-registry")
except metadata.PackageNotFoundError:
return "unknown"
def _git_value(root: Path, *args: str) -> str | None:
try:
result = subprocess.run(
["git", "-C", str(root), *args],
check=False,
capture_output=True,
text=True,
)
except OSError:
return None
value = result.stdout.strip()
return value if result.returncode == 0 and value else None
def _dirty_state(root: Path) -> str:
if not (root / ".git").exists():
return "unknown"
try:
result = subprocess.run(
["git", "-C", str(root), "status", "--short"],
check=False,
capture_output=True,
text=True,
)
except OSError:
return "unknown"
if result.returncode != 0:
return "unknown"
return "dirty" if result.stdout.strip() else "clean"
def _comparison_eligibility(role: str, release_binding_status: str) -> str:
if role == "negative_regression_seed":
return "eligible_as_negative_seed"
if release_binding_status == "complete":
return "eligible"
return "not_comparable"
def _summary(role: str, regression_patterns: list[dict[str, str]]) -> str:
if role == "negative_regression_seed":
return "Historical run captured as a negative self-scoping regression seed."
if regression_patterns:
return "Generated self-scoping assessment repeats known regression patterns."
return "Generated self-scoping assessment artifact for comparison."
def _rationale(
regression_patterns: list[dict[str, str]],
comparison_eligibility: str,
) -> list[str]:
rationale: list[str] = []
if comparison_eligibility == "not_comparable":
rationale.append("Engine identity is incomplete, so this cannot be a comparable baseline.")
for pattern in regression_patterns:
rationale.append(f"{pattern['id']}: {pattern['description']}")
return rationale
def _fact_summary(facts: list[ObservedFact]) -> dict[str, Any]:
return {
"counts_by_kind": dict(sorted(Counter(fact.kind for fact in facts).items())),
"contamination_sources": _contamination_sources(facts),
}
def _contamination_sources(facts: list[ObservedFact]) -> list[dict[str, str]]:
provider_kinds = {
"llm_provider",
"credential_config",
"provider_registry",
"fallback_policy",
}
suspicious_segments = (
"test",
"tests/",
"fixtures",
"expectations",
"schemas.py",
"scanner.py",
"normalization.py",
"workplans/",
)
results: list[dict[str, str]] = []
seen: set[str] = set()
for fact in facts:
lower = fact.path.lower()
if fact.kind not in provider_kinds or not any(segment in lower for segment in suspicious_segments):
continue
if fact.path in seen:
continue
seen.add(fact.path)
results.append(
{
"path": fact.path,
"reason": (
"Provider-related fact came from scanner rules, tests, fixtures, "
"schemas, or workplan context and needs native-utility review."
),
}
)
return sorted(results, key=lambda item: item["path"])
def _content_chunk_summary(chunks: list[ContentChunk]) -> dict[str, Any]:
source_roles = Counter(
str(chunk.metadata.get("source_role", "") or "unknown") for chunk in chunks
)
return {
"total": len(chunks),
"counts_by_kind": dict(sorted(Counter(chunk.kind for chunk in chunks).items())),
"counts_by_source_role": dict(sorted(source_roles.items())),
"paths": sorted({chunk.path for chunk in chunks}),
}
def _candidate_ability(ability: CandidateAbility) -> dict[str, Any]:
return {
"name": ability.name,
"status": ability.status,
"primary_class": ability.primary_class,
"source_refs": [_source_ref(ref) for ref in ability.source_refs],
"capabilities": [
_candidate_capability(capability) for capability in ability.capabilities
],
}
def _candidate_capability(capability: CandidateCapability) -> dict[str, Any]:
return {
"name": capability.name,
"status": capability.status,
"primary_class": capability.primary_class,
"source_refs": [_source_ref(ref) for ref in capability.source_refs],
"features": [_candidate_feature(feature) for feature in capability.features],
"evidence": [_candidate_evidence(evidence) for evidence in capability.evidence],
}
def _candidate_feature(feature: CandidateFeature) -> dict[str, Any]:
return {
"name": feature.name,
"type": feature.type,
"status": feature.status,
"primary_class": feature.primary_class,
"location": feature.location,
"source_refs": [_source_ref(ref) for ref in feature.source_refs],
}
def _candidate_evidence(evidence: CandidateEvidence) -> dict[str, Any]:
return {
"type": evidence.type,
"reference": evidence.reference,
"strength": evidence.strength,
"status": evidence.status,
"source_refs": [_source_ref(ref) for ref in evidence.source_refs],
}
def _approved_map(ability_map: RepositoryAbilityMap) -> dict[str, Any]:
return {
"scope": asdict(ability_map.scope),
"abilities": [_approved_ability(ability) for ability in ability_map.abilities],
}
def _approved_ability(ability: Ability) -> dict[str, Any]:
return {
"name": ability.name,
"primary_class": ability.primary_class,
"capabilities": [
{
"name": capability.name,
"primary_class": capability.primary_class,
"features": [
{
"name": feature.name,
"type": feature.type,
"primary_class": feature.primary_class,
"location": feature.location,
"source_refs": [
_source_ref(ref) for ref in feature.source_refs
],
}
for feature in capability.features
],
"evidence": [asdict(evidence) for evidence in capability.evidence],
}
for capability in ability.capabilities
],
}
def _source_ref(ref: SourceReference) -> dict[str, Any]:
return asdict(ref)
def _review_decision(decision: ReviewDecision) -> dict[str, Any]:
payload = asdict(decision)
payload["quality_criteria_version"] = active_quality_criteria_version()
return payload
def _known_regression_patterns(
abilities: list[CandidateAbility],
decisions: list[ReviewDecision],
) -> list[dict[str, str]]:
patterns: list[dict[str, str]] = []
llm_capabilities = [
capability
for ability in abilities
for capability in ability.capabilities
if capability.name == KNOWN_PROVIDER_ROUTING_CAPABILITY
]
if llm_capabilities:
patterns.append(
{
"id": "RREG-SELF-REG-001",
"title": "LLM provider vocabulary promoted as native capability",
"severity": "critical",
"description": (
"Generated tree contains Route LLM Requests Across Providers "
"as a repo-scoping capability."
),
"detection_hint": (
"Flag the provider-routing capability unless product intent "
"and public implementation explicitly support it."
),
}
)
if any(
feature.type in {"API", "CLI"}
for capability in llm_capabilities
for feature in capability.features
):
patterns.append(
{
"id": "RREG-SELF-REG-002",
"title": "Native API and CLI surfaces attached under false capability",
"severity": "high",
"description": (
"API or CLI surface features are nested below provider routing."
),
"detection_hint": (
"Flag API/CLI surface features whose parent capability is "
"llm-integration or provider-routing."
),
}
)
if any(decision.action == "trusted_auto_approve_candidate_graph" for decision in decisions):
patterns.append(
{
"id": "RREG-SELF-REG-003",
"title": "Deterministic trusted auto-approval accepted candidate truth",
"severity": "high",
"description": (
"Candidate characteristics were approved through trusted "
"auto-approval instead of human or agentic judgement."
),
"detection_hint": "Flag trusted_auto_approve_candidate_graph review decisions.",
}
)
return patterns
def _execution_mode(decisions: list[ReviewDecision]) -> str:
if any(decision.action.startswith("agentic_review") for decision in decisions):
return "agentic-review"
if any(decision.action == "trusted_auto_approve_candidate_graph" for decision in decisions):
return "trusted-auto-review"
if any(decision.action == "llm_extraction_used" for decision in decisions):
return "llm-assisted"
if any(decision.action.startswith("approve") for decision in decisions):
return "manual-review"
return "deterministic-only"
def _candidate_source(decisions: list[ReviewDecision]) -> str:
return "llm+deterministic" if any(
decision.action == "llm_extraction_used" for decision in decisions
) else "deterministic"
def _acceptance_mode(decisions: list[ReviewDecision]) -> str:
agentic_decision = next(
(decision for decision in decisions if decision.action.startswith("agentic_review")),
None,
)
if agentic_decision is not None:
return agentic_decision.action
if any(decision.action == "trusted_auto_approve_candidate_graph" for decision in decisions):
return "trusted_auto_approve_candidate_graph"
if any(decision.action == "approve_candidate_graph" for decision in decisions):
return "manual_candidate_graph_approval"
if any(decision.action == "approve_analysis_run_changes" for decision in decisions):
return "manual_change_approval"
return "pending_review"
def _timestamp(value: str | None) -> str | None:
if value is None:
return None
if "T" in value:
return value
return value.replace(" ", "T") + "Z"
def _artifact_id(repository_name: str, analysis_run_id: int, role: str) -> str:
return f"{_slug(repository_name)}-{role}-run-{analysis_run_id}"
def _slug(value: str) -> str:
return "-".join(
token for token in "".join(char.lower() if char.isalnum() else "-" for char in value).split("-") if token
)
def artifact_json(artifact: dict[str, Any]) -> str:
return json.dumps(artifact, indent=2, sort_keys=True) + "\n"

238
src/repo_scoping/self_scoping/comparison.py Normal file
View file

@ -0,0 +1,238 @@
from __future__ import annotations
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
COMPARISON_SCHEMA_VERSION = "self-scoping-comparison/v1"
def load_json(path: str | Path) -> dict[str, Any]:
return json.loads(Path(path).read_text(encoding="utf-8"))
def compare_assessment_to_golden(
golden_profile: dict[str, Any],
assessment: dict[str, Any],
) -> dict[str, Any]:
expected = _expected_capabilities(golden_profile)
forbidden = _forbidden_capabilities(golden_profile)
generated = _generated_capabilities(assessment)
generated_names = set(generated)
missing_expected = sorted(expected - generated_names)
matched_expected = sorted(expected & generated_names)
forbidden_present = sorted(forbidden & generated_names)
known_regressions = assessment.get("known_regression_patterns", [])
misplaced_features = _misplaced_features(generated)
status = _status(
missing_expected=missing_expected,
forbidden_present=forbidden_present,
known_regressions=known_regressions,
misplaced_features=misplaced_features,
)
return {
"schema_version": COMPARISON_SCHEMA_VERSION,
"comparison_id": _comparison_id(golden_profile, assessment),
"created_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"golden_profile_id": golden_profile.get("profile_id", ""),
"assessment_artifact_id": assessment.get("artifact_id", ""),
"target_repo_slug": assessment.get("target_repository", {}).get("repo_slug", ""),
"status": status,
"summary": _summary(status, missing_expected, forbidden_present, known_regressions),
"matched_expected_capabilities": matched_expected,
"missing_expected_capabilities": missing_expected,
"unexpected_native_capabilities": _unexpected_capabilities(
generated_names,
expected,
forbidden,
),
"forbidden_native_capabilities_present": forbidden_present,
"known_regression_patterns": known_regressions,
"misplaced_features": misplaced_features,
"comparison_hints": _comparison_hints(status),
}
def comparison_json(comparison: dict[str, Any]) -> str:
return json.dumps(comparison, indent=2, sort_keys=True) + "\n"
def comparison_markdown(comparison: dict[str, Any]) -> str:
lines = [
f"# Self-Scoping Comparison: {comparison['assessment_artifact_id']}",
"",
f"- Status: `{comparison['status']}`",
f"- Golden profile: `{comparison['golden_profile_id']}`",
f"- Target repo: `{comparison['target_repo_slug']}`",
f"- Summary: {comparison['summary']}",
"",
"## Missing Expected Capabilities",
*_bullets(comparison["missing_expected_capabilities"]),
"",
"## Forbidden Native Capabilities Present",
*_bullets(comparison["forbidden_native_capabilities_present"]),
"",
"## Known Regression Patterns",
*_regression_bullets(comparison["known_regression_patterns"]),
"",
"## Misplaced Features",
*_misplaced_feature_bullets(comparison["misplaced_features"]),
"",
"## Matched Expected Capabilities",
*_bullets(comparison["matched_expected_capabilities"]),
"",
"## Review Hints",
*_bullets(comparison["comparison_hints"]),
"",
]
return "\n".join(lines)
def _expected_capabilities(golden_profile: dict[str, Any]) -> set[str]:
return {
capability["name"]
for capability in golden_profile.get("ability", {}).get("expected_capabilities", [])
if capability.get("name")
}
def _forbidden_capabilities(golden_profile: dict[str, Any]) -> set[str]:
return {
capability["name"]
for capability in golden_profile.get("forbidden_native_capabilities", [])
if capability.get("name")
}
def _generated_capabilities(assessment: dict[str, Any]) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
for ability in assessment.get("generated_tree", {}).get("abilities", []):
for capability in ability.get("capabilities", []):
name = capability.get("name")
if name:
result[name] = capability
return result
def _unexpected_capabilities(
generated_names: set[str],
expected: set[str],
forbidden: set[str],
) -> list[str]:
return sorted(generated_names - expected - forbidden)
def _misplaced_features(
generated: dict[str, dict[str, Any]],
) -> list[dict[str, str]]:
misplaced: list[dict[str, str]] = []
for capability_name, capability in generated.items():
primary_class = capability.get("primary_class", "")
if primary_class not in {"llm-integration", "provider-routing"}:
continue
for feature in capability.get("features", []):
if feature.get("type") not in {"API", "CLI"}:
continue
misplaced.append(
{
"capability": capability_name,
"feature": feature.get("name", ""),
"feature_type": feature.get("type", ""),
"reason": "API/CLI surface is nested below provider-routing capability.",
}
)
return misplaced
def _status(
*,
missing_expected: list[str],
forbidden_present: list[str],
known_regressions: list[dict[str, Any]],
misplaced_features: list[dict[str, str]],
) -> str:
if forbidden_present or misplaced_features or any(
item.get("severity") in {"high", "critical"} for item in known_regressions
):
return "regression"
if missing_expected or known_regressions:
return "needs_review"
return "candidate_improvement"
def _summary(
status: str,
missing_expected: list[str],
forbidden_present: list[str],
known_regressions: list[dict[str, Any]],
) -> str:
if status == "regression":
return (
"Assessment repeats known or forbidden self-scoping patterns; prefer "
"the golden profile until the engine is corrected."
)
if status == "needs_review":
return (
f"Assessment needs review: {len(missing_expected)} expected "
f"capability(s) missing and {len(known_regressions)} regression "
"pattern(s) reported."
)
return "Assessment covers the golden profile without known regression patterns."
def _comparison_hints(status: str) -> list[str]:
if status == "regression":
return [
"Do not promote this assessment as a preferred baseline.",
"Inspect forbidden capabilities and misplaced features first.",
"Use the findings as signal for scanner, generator, or acceptance-policy changes.",
]
if status == "needs_review":
return [
"Review missing expected capabilities before choosing old or new output.",
"Check whether the golden profile needs a curator-approved update.",
]
return [
"Candidate appears better than the known golden checks.",
"Human or agentic review should still confirm source evidence quality.",
]
def _comparison_id(
golden_profile: dict[str, Any],
assessment: dict[str, Any],
) -> str:
return (
f"{golden_profile.get('profile_id', 'golden')}"
f"__{assessment.get('artifact_id', 'assessment')}"
)
def _bullets(items: list[str]) -> list[str]:
if not items:
return ["- None"]
return [f"- {item}" for item in items]
def _regression_bullets(items: list[dict[str, Any]]) -> list[str]:
if not items:
return ["- None"]
return [
f"- `{item.get('id', '')}` {item.get('title', '')}: {item.get('description', '')}"
for item in items
]
def _misplaced_feature_bullets(items: list[dict[str, str]]) -> list[str]:
if not items:
return ["- None"]
return [
(
f"- `{item['feature']}` under `{item['capability']}` "
f"({item['feature_type']}): {item['reason']}"
)
for item in items
]

217
src/repo_scoping/self_scoping/review_store.py Normal file
View file

@ -0,0 +1,217 @@
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
SELF_SCOPING_ROOT_ENV = "REPO_REGISTRY_SELF_SCOPING_ROOT"
OUTCOME_SCHEMA_VERSION = "self-scoping-review-outcome/v1"
ALLOWED_OUTCOMES = {
"prefer_golden",
"prefer_assessment",
"prefer_baseline",
"prefer_challenger",
"tie",
"needs_human",
"reject_assessment",
"reject_challenger",
}
@dataclass(frozen=True)
class ReviewArtifact:
path: str
artifact_id: str
title: str
updated_at: str
def self_scoping_root(root: str | Path | None = None) -> Path:
configured = root or os.environ.get(SELF_SCOPING_ROOT_ENV) or "docs/self-scoping"
return Path(configured).resolve()
def list_golden_profiles(root: str | Path | None = None) -> list[ReviewArtifact]:
return _list_artifacts("golden", root=root)
def list_assessment_artifacts(root: str | Path | None = None) -> list[ReviewArtifact]:
return _list_artifacts("assessments", root=root)
def load_json_artifact(
relative_path: str,
root: str | Path | None = None,
) -> dict[str, Any]:
artifact_path = _safe_artifact_path(relative_path, root=root)
return json.loads(artifact_path.read_text(encoding="utf-8"))
def list_outcome_records(root: str | Path | None = None) -> list[dict[str, Any]]:
outcomes_dir = self_scoping_root(root) / "outcomes"
if not outcomes_dir.exists():
return []
records: list[dict[str, Any]] = []
for path in sorted(outcomes_dir.glob("*.json"), reverse=True):
try:
records.append(json.loads(path.read_text(encoding="utf-8")))
except json.JSONDecodeError:
continue
return records
def record_assessment_outcome(
*,
golden_path: str,
assessment_path: str,
outcome: str,
reviewer: str,
notes: str,
comparison_status: str,
root: str | Path | None = None,
) -> dict[str, Any]:
if outcome not in ALLOWED_OUTCOMES:
raise ValueError(f"unsupported review outcome: {outcome}")
base = self_scoping_root(root)
golden = load_json_artifact(golden_path, root=base)
assessment = load_json_artifact(assessment_path, root=base)
created_at = _created_at()
outcome_id = _outcome_id(created_at, assessment_path, outcome)
record = {
"schema_version": OUTCOME_SCHEMA_VERSION,
"outcome_id": outcome_id,
"created_at": created_at,
"reviewer": reviewer.strip() or "codex",
"outcome": outcome,
"notes": notes.strip(),
"comparison_status": comparison_status,
"golden_profile_path": golden_path,
"golden_profile_id": golden.get("profile_id", ""),
"assessment_artifact_path": assessment_path,
"assessment_artifact_id": assessment.get("artifact_id", ""),
"engine_identity": assessment.get("engine_identity", {}),
"decision_scope": "baseline-comparison",
}
_write_outcome(record, base)
return record
def record_assessment_pair_outcome(
*,
baseline_path: str,
challenger_path: str,
outcome: str,
reviewer: str,
notes: str,
comparison_status: str,
root: str | Path | None = None,
) -> dict[str, Any]:
if outcome not in ALLOWED_OUTCOMES:
raise ValueError(f"unsupported review outcome: {outcome}")
base = self_scoping_root(root)
baseline = load_json_artifact(baseline_path, root=base)
challenger = load_json_artifact(challenger_path, root=base)
created_at = _created_at()
outcome_id = _outcome_id(
created_at,
f"{Path(baseline_path).stem}__{Path(challenger_path).stem}",
outcome,
)
record = {
"schema_version": OUTCOME_SCHEMA_VERSION,
"outcome_id": outcome_id,
"created_at": created_at,
"reviewer": reviewer.strip() or "codex",
"outcome": outcome,
"notes": notes.strip(),
"comparison_status": comparison_status,
"baseline_assessment_path": baseline_path,
"baseline_assessment_artifact_id": baseline.get("artifact_id", ""),
"baseline_engine_identity": baseline.get("engine_identity", {}),
"challenger_assessment_path": challenger_path,
"challenger_assessment_artifact_id": challenger.get("artifact_id", ""),
"challenger_engine_identity": challenger.get("engine_identity", {}),
"decision_scope": "assessment-pair-comparison",
}
_write_outcome(record, base)
return record
def _created_at() -> str:
return (
datetime.now(UTC)
.replace(microsecond=0)
.isoformat()
.replace("+00:00", "Z")
)
def _write_outcome(record: dict[str, Any], base: Path) -> None:
outcomes_dir = base / "outcomes"
outcomes_dir.mkdir(parents=True, exist_ok=True)
output_path = outcomes_dir / f"{record['outcome_id']}.json"
output_path.write_text(
json.dumps(record, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def _list_artifacts(kind: str, root: str | Path | None = None) -> list[ReviewArtifact]:
base = self_scoping_root(root)
artifacts: list[ReviewArtifact] = []
for path in sorted((base / kind).glob("*.json")):
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
artifacts.append(
ReviewArtifact(
path=path.relative_to(base).as_posix(),
artifact_id=str(
payload.get("artifact_id") or payload.get("profile_id") or path.stem
),
title=str(
payload.get("title")
or payload.get("assessment", {}).get("summary")
or payload.get("artifact_type")
or path.stem
),
updated_at=str(
payload.get("updated_at") or payload.get("created_at") or ""
),
)
)
return artifacts
def _safe_artifact_path(relative_path: str, root: str | Path | None = None) -> Path:
base = self_scoping_root(root)
artifact_path = (base / relative_path).resolve()
try:
artifact_path.relative_to(base)
except ValueError as exc:
raise ValueError(f"artifact path escapes self-scoping root: {relative_path}") from exc
if artifact_path.suffix != ".json":
raise ValueError(f"artifact path is not JSON: {relative_path}")
if not artifact_path.exists():
raise FileNotFoundError(relative_path)
return artifact_path
def _outcome_id(created_at: str, assessment_path: str, outcome: str) -> str:
timestamp = (
created_at.replace("-", "")
.replace(":", "")
.replace("T", "-")
.replace("Z", "")
)
assessment_stem = Path(assessment_path).stem.replace(".", "-")
return f"{timestamp}__{assessment_stem}__{outcome}__{uuid4().hex[:8]}"