repo-scoping/docs/self-scoping/assessments/repo-scoping-known-bad-2026-05-15-run-39.json

{
  "schema_version": "self-scoping-assessment/v1",
  "artifact_id": "repo-scoping-known-bad-2026-05-15-run-39",
  "artifact_type": "assessment_run",
  "created_at": "2026-05-15T09:28:48Z",
  "target_repository": {
    "repo_slug": "repo-scoping",
    "repository_id": 16,
    "source": "/home/worsch/repo-scoping/var/checkouts/repo-scoping-8a9c4168485c",
    "target_commit": "00b57d509124789059639fedc724d9314edbb7b2",
    "target_branch": "main",
    "dirty_state": "unknown",
    "file_count": 96
  },
  "engine_identity": {
    "repo_scoping_version": "0.1.0",
    "engine_commit": null,
    "engine_release": null,
    "engine_dirty_state": "unknown",
    "scanner_version": "deterministic-v0.1",
    "candidate_generator_version": "unversioned-pre-self-scoping-baseline",
    "quality_criteria_version": "none",
    "prompt_version": null,
    "release_binding_status": "historical_incomplete",
    "release_binding_note": "This historical database run recorded scanner version and target commit, but not the repo-scoping engine commit or release tag that generated the candidate graph."
  },
  "execution": {
    "mode": "trusted-auto-review",
    "analysis_run_id": 39,
    "candidate_source": "deterministic",
    "acceptance_mode": "trusted_auto_approve_candidate_graph",
    "started_at": "2026-05-15T09:28:47Z",
    "completed_at": "2026-05-15T09:28:48Z"
  },
  "assessment": {
    "role": "negative_regression_seed",
    "outcome": "rejected",
    "summary": "The self-analysis promoted LLM-provider vocabulary into a false native repo-scoping capability and attached API/CLI features below it.",
    "reviewer": "codex",
    "comparison_eligibility": "eligible_as_negative_seed",
    "rationale": [
      "repo-scoping uses llm-connect as optional extraction infrastructure; it does not natively route LLM requests across providers.",
      "Provider names came from scanner rules, normalization tokens, schema examples, tests, fixtures, and workplan text rather than product-facing provider-routing behavior.",
      "The generated tree placed native API and CLI surfaces under the false LLM-provider capability, which makes the feature hierarchy misleading."
    ]
  },
  "fact_summary": {
    "counts_by_kind": {
      "config": 1,
      "credential_config": 13,
      "documentation": 14,
      "fallback_policy": 10,
      "framework": 2,
      "intent": 1,
      "interface": 127,
      "language": 1,
      "llm_provider": 41,
      "manifest": 1,
      "provider_registry": 7,
      "scope": 1,
      "test": 19
    },
    "contamination_sources": [
      {
        "path": "src/repo_registry/repo_scanning/scanner.py",
        "reason": "Provider detector constants, credential hint constants, and fallback/provider-registry scanner logic were treated as repo-owned LLM routing evidence."
      },
      {
        "path": "src/repo_registry/candidate_graph/normalization.py",
        "reason": "Provider names used as distinctive candidate-normalization tokens were treated as implementation evidence for provider support."
      },
      {
        "path": "src/repo_registry/web_api/schemas.py",
        "reason": "An OpenRouter example in an expectation-gap schema was treated as provider evidence."
      },
      {
        "path": "tests/expectations/llm_connect_provider_expectations.json",
        "reason": "A fixture describing llm-connect expectations was treated as repo-scoping product behavior."
      },
      {
        "path": "tests/fixtures.py",
        "reason": "Regression fixture vocabulary was treated as native repo-scoping capability evidence."
      },
      {
        "path": "tests/test_candidate_graph.py",
        "reason": "Unit-test examples for LLM-provider detection were treated as product evidence."
      },
      {
        "path": "tests/test_repository_scanner.py",
        "reason": "Scanner tests for provider facts were treated as native provider-routing evidence."
      }
    ]
  },
  "content_chunk_summary": {
    "total": 0,
    "counts_by_kind": {},
    "counts_by_source_role": {},
    "paths": []
  },
  "generated_tree": {
    "abilities": [
      {
        "name": "Support Repo Registry",
        "status": "approved",
        "primary_class": "repository-intelligence",
        "source_refs": [],
        "capabilities": [
          {
            "name": "Route LLM Requests Across Providers",
            "status": "approved",
            "primary_class": "llm-integration",
            "source_refs": [],
            "features": [
              {
                "name": "Use Anthropic Models",
                "type": "integration",
                "status": "approved",
                "primary_class": "integration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Use Claude Models",
                "type": "integration",
                "status": "approved",
                "primary_class": "integration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Use Gemini Models",
                "type": "integration",
                "status": "approved",
                "primary_class": "integration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Use OpenAI Models",
                "type": "integration",
                "status": "approved",
                "primary_class": "integration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Use OpenRouter Models",
                "type": "integration",
                "status": "approved",
                "primary_class": "integration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Configure LLM Provider Credentials",
                "type": "configuration",
                "status": "approved",
                "primary_class": "configuration",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "Maintain LLM Provider Registry",
                "type": "backend",
                "status": "approved",
                "primary_class": "backend",
                "location": "src/repo_registry/repo_scanning/scanner.py",
                "source_refs": []
              },
              {
                "name": "Apply LLM Provider Fallback Policy",
                "type": "backend",
                "status": "approved",
                "primary_class": "backend",
                "location": "src/repo_registry/repo_scanning/scanner.py",
                "source_refs": []
              },
              {
                "name": "HTTP API surface: possible API surface, GET /health, @app.post(, +43 more",
                "type": "API",
                "status": "approved",
                "primary_class": "API",
                "location": "multiple files",
                "source_refs": []
              },
              {
                "name": "CLI command surface: CLI command build_parser, CLI command make_service",
                "type": "CLI",
                "status": "approved",
                "primary_class": "CLI",
                "location": "multiple files",
                "source_refs": []
              }
            ],
            "evidence": []
          }
        ]
      }
    ]
  },
  "approved_map": {},
  "review_decisions": [
    {
      "id": 21,
      "repository_id": 16,
      "analysis_run_id": 39,
      "action": "trusted_auto_approve_candidate_graph",
      "notes": "Trusted auto-populate mode reviewed candidate graph after deterministic candidate generation. Auto-approved 1 safe candidate capability(s); left 0 for review. Approved: Route LLM Requests Across Providers: eligible LLM utility relationship with source support.",
      "created_at": "2026-05-15 09:28:49"
    }
  ],
  "quality_gate_outcomes": [],
  "known_regression_patterns": [
    {
      "id": "RREG-SELF-REG-001",
      "title": "LLM provider vocabulary promoted as native capability",
      "severity": "critical",
      "description": "Scanner, normalization, schema, fixture, test, or workplan vocabulary becomes the native repo-scoping capability Route LLM Requests Across Providers.",
      "detection_hint": "Flag any top-level/native repo-scoping capability named Route LLM Requests Across Providers unless product intent and public implementation explicitly show provider routing as a repo-scoping feature."
    },
    {
      "id": "RREG-SELF-REG-002",
      "title": "Native API and CLI surfaces attached under false capability",
      "severity": "high",
      "description": "General repo-scoping API/CLI interface features are nested below a capability they do not support.",
      "detection_hint": "Flag API or CLI surface features when their parent capability is llm-integration or provider-routing."
    },
    {
      "id": "RREG-SELF-REG-003",
      "title": "Deterministic trusted auto-approval accepted candidate truth",
      "severity": "high",
      "description": "A deterministic rule path approves candidate characteristics without human or agentic judgement.",
      "detection_hint": "Flag trusted_auto_approve_candidate_graph review decisions in self-scoping assessment artifacts."
    }
  ],
  "notes": [
    "This artifact is a negative regression seed, not a desirable baseline.",
    "The historical run is useful for pattern detection but is not fully release-bound because the engine commit was not recorded in the original analysis metadata."
  ]
}