rein-aharness/rein_aharness/fi_research_brief.py

"""Freedom Intelligence daily research brief via llm-connect.

activity-core owns *when* (Temporal schedule + fi_brief_status).
This rein command owns *execution*: draft brief from playbook context,
commit under freedom-intelligence, post fi_daily_brief for idempotence.

No Claude Code / host coding agent — llm-connect only.
"""

from __future__ import annotations

import json
import os
import re
import subprocess
from dataclasses import dataclass, field
from datetime import date, datetime
from pathlib import Path
from typing import Any, Callable
from zoneinfo import ZoneInfo

from rein_aharness import hub
from rein_aharness.llm_connect_client import (
    LLMConnectClient,
    LLMConnectError,
    get_llm_connect_client,
)

_CONTEXT_FILES = (
    "docs/sources-allowlist.md",
    "docs/daily-brief-playbook.md",
    "inventory/RESERVE-STATUS.md",
    "research/2026-07-24-baseline-field-survey.md",
    "research/2026-07-24-nas-strategic-collection-plan.md",
)
_MAX_FILE_CHARS = 5000
_MAX_GIT_LOG = 12


class FiResearchBriefError(RuntimeError):
    pass


@dataclass
class FiResearchBriefResult:
    ok: bool
    date: str
    path: str | None = None
    wrote: bool = False
    committed: bool = False
    head_after: str = ""
    reason: str = ""
    model_meta: dict[str, Any] = field(default_factory=dict)
    skipped_existing: bool = False
    collection_candidates: int = 0


def _berlin_today() -> date:
    try:
        return datetime.now(ZoneInfo("Europe/Berlin")).date()
    except Exception:
        return date.today()


def brief_path_for(repo: Path, day: date) -> Path:
    return repo / "briefs" / f"{day.year:04d}" / f"{day.month:02d}" / f"{day.isoformat()}.md"


def collect_context(repo: Path, day: date) -> str:
    chunks: list[str] = [
        f"Brief date (Europe/Berlin): {day.isoformat()}\n",
        "Repo: freedom-intelligence — axes A–D AI field research brief.\n",
        "Briefs are DELTAS from prior briefs + baseline, not full resurveys.\n",
    ]
    for rel in _CONTEXT_FILES:
        path = repo / rel
        if not path.is_file():
            chunks.append(f"## {rel}\n(missing)\n")
            continue
        text = path.read_text(encoding="utf-8", errors="replace")
        chunks.append(f"## {rel}\n{_truncate(text, _MAX_FILE_CHARS)}\n")

    briefs_root = repo / "briefs"
    if briefs_root.is_dir():
        prior = sorted(briefs_root.rglob("20*.md"))
        prior = [p for p in prior if p.name != f"{day.isoformat()}.md"]
        names = [str(p.relative_to(repo)) for p in prior[-5:]]
        chunks.append(f"## Recent briefs\n{names}\n")
        if prior:
            last = prior[-1]
            chunks.append(
                f"## Previous brief ({last.relative_to(repo)})\n"
                f"{_truncate(last.read_text(encoding='utf-8', errors='replace'), 3500)}\n"
            )

    try:
        log = _git(repo, "log", f"-{_MAX_GIT_LOG}", "--oneline")
        chunks.append(f"## Recent git log\n{log}\n")
    except FiResearchBriefError:
        chunks.append("## Recent git log\n(unavailable)\n")

    return "\n".join(chunks)


def build_prompt(context: str, day: date) -> str:
    return f"""You write the Freedom Intelligence **daily research brief** for {day.isoformat()}.

You are the lab's automated field sensor. Produce a real delta brief operators can
act on — not an empty template.

Rules:
- Deltas only vs previous brief + baseline (do NOT restate the whole baseline).
- Prefer concrete names, dates, licenses, and size/class when known from context.
- Cover axes when there is signal: A frontier/commercial, B open/local,
  C training/FT, D harness/fleet.
- Flag collection candidates only when license/size/rationale are clear.
- Prefer primary sources; mark unverified claims as "unverified".
- "No material delta" is allowed **only** when you have considered the prior
  brief + allowlist + reserve status and still find nothing. Even then:
  - put a one-line justification in headline_deltas (why empty / what was checked)
  - put at least one lab_implications bullet (what to watch next)
- Never output empty headline_deltas. Never leave all axes empty *and*
  collection_candidates empty *and* lab_implications empty together.
- Output **JSON only** (no markdown fence) with this schema:

{{
  "headline_deltas": ["string", "..."],  // 3–7 bullets preferred; min 1
  "axis_a": [{{"item": "...", "delta": "...", "sources": "...", "lab_relevance": "..."}}],
  "axis_b": [same shape],
  "axis_c": [same shape],
  "axis_d": [same shape],
  "collection_candidates": [
    {{"id": "...", "org": "...", "name": "...", "priority": "high|medium|low",
      "reason": "...", "approx_size": "...", "license": "..."}}
  ],
  "lab_implications": ["string"]  // min 1 bullet
}}

Keep each field short (one line). Empty axis arrays are fine when that axis is quiet.

## Context
{context}
"""


def parse_brief_response(text: str) -> dict[str, Any]:
    cleaned = text.strip()
    if cleaned.startswith("```"):
        cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned)
        cleaned = re.sub(r"\s*```$", "", cleaned)
    try:
        data = json.loads(cleaned)
    except json.JSONDecodeError as exc:
        m = re.search(r"\{.*\}", cleaned, re.S)
        if not m:
            raise FiResearchBriefError(f"LLM response is not JSON: {exc}") from exc
        try:
            data = json.loads(m.group(0))
        except json.JSONDecodeError as exc2:
            raise FiResearchBriefError(f"LLM response is not JSON: {exc2}") from exc2
    if not isinstance(data, dict):
        raise FiResearchBriefError("LLM JSON root must be an object")
    return data


def _ensure_minimum_signal(data: dict[str, Any], day: date) -> dict[str, Any]:
    """Reject pure empty shells so operators do not get green-but-useless briefs."""
    headlines = data.get("headline_deltas") or []
    if isinstance(headlines, str):
        headlines = [headlines]
    headlines = [str(x).strip() for x in headlines if str(x).strip()]

    axes = []
    for key in ("axis_a", "axis_b", "axis_c", "axis_d"):
        rows = data.get(key) or []
        if isinstance(rows, list):
            axes.extend(rows)
    cands = data.get("collection_candidates") or []
    if not isinstance(cands, list):
        cands = []
    impl = data.get("lab_implications") or []
    if isinstance(impl, str):
        impl = [impl]
    impl = [str(x).strip() for x in impl if str(x).strip()]

    emptyish = (
        (not headlines or all("no material delta" in h.lower() for h in headlines))
        and not axes
        and not cands
        and not impl
    )
    if emptyish:
        data = dict(data)
        data["headline_deltas"] = [
            f"No material public delta confirmed for {day.isoformat()} "
            f"from allowlist context + prior briefs (automated scan)."
        ]
        data["lab_implications"] = [
            "Re-check frontier trackers and HF open-weight leaders tomorrow; "
            "reserve plan unchanged until a concrete candidate appears."
        ]
    elif not headlines:
        data = dict(data)
        data["headline_deltas"] = [
            f"Field scan completed for {day.isoformat()} (see axes / implications)."
        ]
    elif not impl:
        data = dict(data)
        data["lab_implications"] = [
            "No change to reserve posture from today's deltas."
        ]
    return data


def render_brief(day: date, data: dict[str, Any]) -> str:
    headlines = data.get("headline_deltas") or ["No material delta."]
    if isinstance(headlines, str):
        headlines = [headlines]
    hl = "\n".join(f"- {str(x).strip().lstrip('- ')}" for x in headlines[:8] if str(x).strip())

    def axis_table(key: str) -> str:
        rows = data.get(key) or []
        if not rows:
            return "*(none)*\n"
        lines = [
            "| Item | Delta | Sources | Lab relevance |",
            "| ---- | ----- | ------- | ------------- |",
        ]
        for row in rows[:12]:
            if not isinstance(row, dict):
                continue
            item = _cell(row.get("item"))
            delta = _cell(row.get("delta"))
            sources = _cell(row.get("sources"))
            lab = _cell(row.get("lab_relevance"))
            lines.append(f"| {item} | {delta} | {sources} | {lab} |")
        return "\n".join(lines) + "\n"

    cands = data.get("collection_candidates") or []
    if not cands:
        cand_md = "*(none)*\n"
    else:
        cand_lines = [
            "| id | org | name | priority | reason | approx size | license |",
            "| -- | --- | ---- | -------- | ------ | ----------- | ------- |",
        ]
        for c in cands[:10]:
            if not isinstance(c, dict):
                continue
            cand_lines.append(
                "| {id} | {org} | {name} | {priority} | {reason} | {approx_size} | {license} |".format(
                    id=_cell(c.get("id")),
                    org=_cell(c.get("org")),
                    name=_cell(c.get("name")),
                    priority=_cell(c.get("priority")),
                    reason=_cell(c.get("reason")),
                    approx_size=_cell(c.get("approx_size")),
                    license=_cell(c.get("license")),
                )
            )
        cand_md = "\n".join(cand_lines) + "\n"

    impl = data.get("lab_implications") or []
    if isinstance(impl, str):
        impl = [impl]
    impl_md = "\n".join(f"- {str(x).strip().lstrip('- ')}" for x in impl[:8] if str(x).strip()) or "- *(none)*"

    return (
        f"---\n"
        f"date: {day.isoformat()}\n"
        f"timezone: Europe/Berlin\n"
        f"author: rein-aharness\n"
        f"status: final\n"
        f"sources_checked:\n"
        f"  - docs/sources-allowlist.md\n"
        f"  - automated field scan via llm-connect\n"
        f"---\n"
        f"\n"
        f"# Freedom Intelligence Daily Brief — {day.isoformat()}\n"
        f"\n"
        f"> Automated brief (activity-core schedule → rein-aharness). "
        f"Deltas only.\n"
        f"\n"
        f"## Headline deltas\n"
        f"\n"
        f"{hl}\n"
        f"\n"
        f"## Frontier & commercial (axis A)\n"
        f"\n"
        f"{axis_table('axis_a')}\n"
        f"## Edge / local / open (axis B)\n"
        f"\n"
        f"{axis_table('axis_b')}\n"
        f"## Training & specialization (axis C)\n"
        f"\n"
        f"{axis_table('axis_c')}\n"
        f"## Harness & fleet (axis D)\n"
        f"\n"
        f"{axis_table('axis_d')}\n"
        f"## Collection candidates\n"
        f"\n"
        f"{cand_md}\n"
        f"## Lab implications\n"
        f"\n"
        f"{impl_md}\n"
    )


def _cell(value: Any) -> str:
    s = str(value or "").replace("|", "/").replace("\n", " ").strip()
    return s[:200] if s else ""


def _truncate(text: str, n: int) -> str:
    if len(text) <= n:
        return text
    return text[: n - 20] + "\n…(truncated)…\n"


def _git(repo: Path, *args: str) -> str:
    result = subprocess.run(
        ["git", "-C", str(repo), *args],
        capture_output=True,
        text=True,
        timeout=60,
    )
    if result.returncode != 0:
        raise FiResearchBriefError(
            f"git {' '.join(args)} failed: {result.stderr.strip()[:200]}"
        )
    return result.stdout.strip()


def run_fi_research_brief(
    target_repo: Path,
    *,
    day: date | None = None,
    force: bool = False,
    report_to_hub: bool = True,
    commit: bool = True,
    client: LLMConnectClient | None = None,
    complete_fn: Callable[[str], str] | None = None,
) -> FiResearchBriefResult:
    repo = target_repo.expanduser().resolve()
    day = day or _berlin_today()
    path = brief_path_for(repo, day)

    if path.is_file() and not force:
        result = FiResearchBriefResult(
            ok=True,
            date=day.isoformat(),
            path=str(path.relative_to(repo)),
            skipped_existing=True,
            reason="brief already exists for today",
        )
        try:
            result.head_after = _git(repo, "rev-parse", "HEAD")
        except FiResearchBriefError:
            pass
        _hub(result, report_to_hub, repo)
        return result

    context = collect_context(repo, day)
    prompt = build_prompt(context, day)
    meta: dict[str, Any] = {}

    try:
        if complete_fn is not None:
            content = complete_fn(prompt)
        else:
            llm = client or get_llm_connect_client()
            model = (
                os.environ.get("FI_RESEARCH_BRIEF_MODEL", "").strip()
                or os.environ.get("BRIEF_DAILY_MODEL", "").strip()
                or os.environ.get("MAIL_TRIAGE_MODEL", "").strip()
            )
            content = llm.complete(
                prompt,
                model=model,
                config={
                    "temperature": float(
                        os.environ.get("FI_RESEARCH_BRIEF_TEMPERATURE", "0.3")
                    ),
                    "max_tokens": int(
                        os.environ.get("FI_RESEARCH_BRIEF_MAX_TOKENS", "4000")
                    ),
                },
            )
            meta = dict(llm.last_response_metadata or {})
        data = parse_brief_response(content)
        data = _ensure_minimum_signal(data, day)
        markdown = render_brief(day, data)
        n_cands = len(data.get("collection_candidates") or [])
    except (LLMConnectError, FiResearchBriefError, OSError) as exc:
        result = FiResearchBriefResult(
            ok=False,
            date=day.isoformat(),
            reason=str(exc)[:300],
            model_meta=meta,
        )
        _hub(result, report_to_hub, repo)
        return result

    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(markdown, encoding="utf-8")
    rel = str(path.relative_to(repo))

    committed = False
    head_after = ""
    if commit:
        try:
            _git(repo, "add", rel)
            status = _git(repo, "status", "--porcelain", rel)
            if status.strip():
                _git(
                    repo,
                    "commit",
                    "-m",
                    f"FI daily research brief {day.isoformat()} (activity-core rhythm)",
                )
                committed = True
            head_after = _git(repo, "rev-parse", "HEAD")
            # Best-effort push so workstation / Forgejo see the brief.
            # Diverged branches must not fail the run; log via reason only if push fails
            # after a successful write.
            if committed and os.environ.get("FI_RESEARCH_BRIEF_PUSH", "1").strip().lower() not in {
                "0",
                "false",
                "no",
                "off",
            }:
                try:
                    _git(repo, "push", "origin", "HEAD")
                except FiResearchBriefError:
                    # Leave brief committed locally; operators reconcile git separately.
                    pass
        except FiResearchBriefError as exc:
            result = FiResearchBriefResult(
                ok=False,
                date=day.isoformat(),
                path=rel,
                wrote=True,
                reason=f"write ok but commit failed: {exc}",
                model_meta=meta,
                collection_candidates=n_cands,
            )
            _hub(result, report_to_hub, repo)
            return result
    else:
        try:
            head_after = _git(repo, "rev-parse", "HEAD")
        except FiResearchBriefError:
            pass

    result = FiResearchBriefResult(
        ok=True,
        date=day.isoformat(),
        path=rel,
        wrote=True,
        committed=committed,
        head_after=head_after,
        model_meta=meta,
        collection_candidates=n_cands,
    )
    _hub(result, report_to_hub, repo)
    return result


def _hub(result: FiResearchBriefResult, report_to_hub: bool, repo: Path) -> None:
    if not report_to_hub:
        return
    if result.ok:
        # Idempotence for activity-core fi_brief_status resolver
        event_type = "fi_daily_brief"
        summary = (
            f"FI daily brief {result.date}"
            + (
                " (already present)"
                if result.skipped_existing
                else f" wrote={result.wrote} committed={result.committed}"
            )
        )
        detail = {
            "repo": "freedom-intelligence",
            "date": result.date,
            "path": result.path,
            "collection_candidates": result.collection_candidates,
            "wrote": result.wrote,
            "committed": result.committed,
            "skipped_existing": result.skipped_existing,
            "executor": "rein-aharness",
        }
    else:
        event_type = "executor_run"
        summary = f"FI daily research brief failed: {result.reason}"
        detail = {
            "repo": "freedom-intelligence",
            "ok": False,
            "date": result.date,
            "reason": result.reason,
            "model_meta": result.model_meta,
            "executor": "rein-aharness",
        }
    hub.post_progress_event(summary=summary, event_type=event_type, detail=detail)