"""Retain automation briefs that inbound-library routed to this repository. The library is reached only through its pull CLI (`new`, `show`, `get`). Grok mails a notification rather than the brief: the automation name, the title, a teaser cut off mid-sentence, and a "Continue reading" link that needs a signed-in session. Such a mail is retained as a `partial` brief, linked to its period. The full text still arrives through the drop folder (external_briefs). No session is sent to grok.com. """ import hashlib import json import re import subprocess import sys from datetime import date, datetime from pathlib import Path from zoneinfo import ZoneInfo from .ingest import MAX_BYTES from .store import all_records, atomic, put AUTOMATION = re.compile(r'intelligence-radar-(daily|weekly)', re.IGNORECASE) CONTINUE = re.compile(r'Continue reading\s*?\s', re.IGNORECASE) FORWARD_MARKER = re.compile(r'-{3,}\s*(?:Forwarded message|Weitergeleitete Nachricht)\s*-{3,}', re.IGNORECASE) FORWARD_DATE = re.compile(r'^\s*(?:Date|Datum)\s*:\s*(.+)$', re.MULTILINE) MONTHS = {'jan': 1, 'feb': 2, 'mar': 3, 'mär': 3, 'maer': 3, 'apr': 4, 'may': 5, 'mai': 5, 'jun': 6, 'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'okt': 10, 'nov': 11, 'dec': 12, 'dez': 12} class LibraryUnavailable(Exception): pass def library_cli(spec, *args): """Run one inbound-library pull command; return its stdout bytes.""" root = Path(spec['root']).expanduser() if not (root / 'inbound_library').is_dir(): raise LibraryUnavailable(f'inbound-library not found at {root}') done = subprocess.run([sys.executable, '-m', 'inbound_library', '--root', str(root), *args], cwd=root, capture_output=True, timeout=60) if done.returncode: raise LibraryUnavailable(f"inbound-library {args[0]} exited {done.returncode}") return done.stdout def original_date(text): """Send date of the forwarded original ("So., 4. Okt. 2026 um 20:07 Uhr" or "Sun, Oct 4, 2026 at 8:07 PM"), or None.""" marker = FORWARD_MARKER.search(text or '') match = FORWARD_DATE.search(text, marker.end()) if marker else None if not match: return None value = match.group(1) german = re.search(r'(\d{1,2})\.\s*([A-Za-zäÄ]+)\.?\s+(\d{4})', value) english = re.search(r'([A-Za-z]{3})[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4})', value) try: if german: day, month, year = german.groups() return date(int(year), MONTHS[month[:3].lower()], int(day)) if english: month, day, year = english.groups() return date(int(year), MONTHS[month[:3].lower()], int(day)) except (KeyError, ValueError): return None return None def classify(text, received_at, zone='Europe/Berlin'): """Cadence and period of a mailed brief. Returns None when it is not one.""" automation = AUTOMATION.search(text or '') if not automation: return None cadence = automation.group(1).lower() sent = original_date(text) basis = 'forwarded_date' if sent is None: sent = datetime.fromisoformat(received_at).astimezone(ZoneInfo(zone)).date() basis = 'received_at' if cadence == 'weekly': iso = sent.isocalendar() period = f'{iso.year}-W{iso.week:02}' else: period = sent.isoformat() continued = CONTINUE.search(text + '\n') return {'cadence': cadence, 'period': period, 'filename_date': sent.isoformat(), 'period_basis': basis, 'completeness': 'partial' if continued else 'complete', 'continue_url': continued.group(1) if continued else ''} def pull(root, spec, *, cli=library_cli, now=None, zone='Europe/Berlin'): """Retain every routed brief from the configured library sources.""" root = Path(root) moment = now or datetime.now(ZoneInfo(zone)) try: notices = json.loads(cli(spec, 'new', '--repo', spec.get('repo', 'intelligence-radar'))) except (LibraryUnavailable, OSError, ValueError, subprocess.TimeoutExpired) as exc: return {'ok': False, 'status': 'library_unavailable', 'error': str(exc), 'items': []} seen = {record['content_sha256'] for _, record in all_records(root, 'external-briefs')} \ if (root / 'data' / 'external-briefs').is_dir() else set() items = [] for notice in notices: provider = spec.get('sources', {}).get(notice['source']) if provider is None: continue item = notice['item'] if item in seen: items.append({'item': item, 'status': 'duplicate'}) continue try: items.append(_retain(root, spec, cli, notice, provider, moment.isoformat(), zone)) except (LibraryUnavailable, OSError, ValueError, subprocess.TimeoutExpired) as exc: items.append({'item': item, 'status': 'error', 'error': str(exc)}) continue seen.add(item) return {'ok': not any(i['status'] == 'error' for i in items), 'status': 'pulled', 'items': items} def _retain(root, spec, cli, notice, provider, imported_at, zone): item = notice['item'] shown = json.loads(cli(spec, 'show', item)) derived = shown.get('derived') or {} text = derived.get('text', '') parsed = classify(text, notice['received_at'], zone) if parsed is None: return {'item': item, 'status': 'not_a_brief'} raw = cli(spec, 'get', item) if hashlib.sha256(raw).hexdigest() != item: raise ValueError(f'library item {item[:12]} does not match its id') if len(raw) > MAX_BYTES: return {'item': item, 'status': 'too_large'} subject = derived.get('subject', '') relative = _write_markdown(root, provider, parsed, text, item, imported_at, subject, notice['received_at']) atomic(root / 'data' / 'artifacts' / item, raw, immutable=True) record = {'schema_version': 2, 'provider': provider, 'cadence': parsed['cadence'], 'period': parsed['period'], 'filename_date': parsed['filename_date'], 'source_name': subject, 'content_sha256': item, 'digest_path': relative, 'imported_at': imported_at, 'origin': 'inbound-library', 'library_source': notice['source'], 'received_at': notice['received_at'], 'period_basis': parsed['period_basis'], 'completeness': parsed['completeness'], 'continue_url': parsed['continue_url']} return {'item': item, 'status': 'retained', 'id': put(root, 'external-briefs', record), 'digest_path': relative, 'cadence': parsed['cadence'], 'period': parsed['period'], 'completeness': parsed['completeness']} def _write_markdown(root, provider, parsed, text, item, imported_at, subject, received_at): folder = root / 'digests' / 'external' / parsed['cadence'] folder.mkdir(parents=True, exist_ok=True) # The stable -.md name belongs to the full brief from the drop folder. target = folder / f"{parsed['period']}-{provider}-mail-{item[:12]}.md" if not target.exists(): label = ('Partial: mailed teaser, cut off by the provider. The full brief needs the ' 'drop folder.' if parsed['completeness'] == 'partial' else 'Complete mailed brief.') body = '\n'.join([ '> External automation brief received by email through inbound-library. Unverified. ' 'Not a reviewed digest.', f'> {label}', '', f'- Provider: {provider}', f"- Cadence: {parsed['cadence']}", f"- Period: {parsed['period']} (from {parsed['period_basis']})", f'- Subject: {subject}', f'- Received: {received_at}', f'- Library item (SHA-256 of the original message): {item}', f"- Continue reading (needs sign-in, not fetched): {parsed['continue_url'] or '-'}", f'- Imported: {imported_at}', '', text if text.endswith('\n') else text + '\n']) target.write_text(body, encoding='utf-8') return target.relative_to(root).as_posix()