From 65887ef637d7daae7cf7893756472e61fc6e8a33 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 5 Oct 2026 16:24:33 +0200 Subject: [PATCH] Route forwarded mail by its original sender; register Grok briefs via founder. Adds forwarded_from (original sender from the forwarded header block, English/German) to delivery provenance and source match rules, so only founder forwards of noreply@x.ai mail route as Grok briefs. Co-Authored-By: Claude Opus 5.5 Assistant: claude-code Assistant-Model: opus Assistant-Process: 1170398@bnt-lap001 Assistant-Session: 622cd705-cb31-4322-ba08-dd42e44fa583 --- docs/data-contracts.md | 3 +- inbound_library/intake.py | 28 ++++++++++++++++--- inbound_library/registry.py | 8 ++++-- registry/sources/grok-briefs-via-founder.json | 13 +++++++++ tests/test_library.py | 16 +++++++++++ 5 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 registry/sources/grok-briefs-via-founder.json diff --git a/docs/data-contracts.md b/docs/data-contracts.md index 8a264da..852ed8a 100644 --- a/docs/data-contracts.md +++ b/docs/data-contracts.md @@ -41,6 +41,7 @@ One record per delivery of an item. | `received_at` | IMAP INTERNALDATE (UTC ISO 8601) | | `mailbox` | `/`; for `imap` only | | `message_id`, `sender`, `subject`, `list_id` | for `imap` only; `sender` is lowercased | +| `forwarded_from` | for `imap`: the original sender from the first forwarded header block in the body (`From:`/`Von:` after a forward marker), or `''` | | `authentication_results` | raw headers, recorded as provenance, not as trust | | `parent_item`, `filename`, `content_type` | for `attachment` only | @@ -69,7 +70,7 @@ subscriptions may only use registered topics. | `id` | the file name without `.json` | | `owner_repo` | the repository that registered it | | `channel` | `email` or `url` | -| `match` | for `email`: `from` (required), `subject` (substring), `list_id` | +| `match` | for `email`: `from` (required), `subject` (substring), `list_id`, `forwarded_from` (the original sender of a forwarded mail) | | `url` | for `url` | | `topics` | list of registered topics | | `cadence` | `daily`, `weekly` or `irregular`; daily and weekly email sources are watched for silence (36 h or 9 days) | diff --git a/inbound_library/intake.py b/inbound_library/intake.py index 2575b3b..89a2685 100644 --- a/inbound_library/intake.py +++ b/inbound_library/intake.py @@ -107,6 +107,22 @@ def _text(message): return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip() +# Header line of a forwarded message, as Gmail and common clients write it in the +# body, in English or German. +FORWARDED_FROM = re.compile(r'^\s*(?:From|Von)\s*:\s*(.+)$', re.MULTILINE) +FORWARD_MARKER = re.compile(r'-{3,}\s*(?:Forwarded message|Weitergeleitete Nachricht)\s*-{3,}', + re.IGNORECASE) + + +def forwarded_sender(text): + """Original sender of a forwarded mail, from the first header block after the marker.""" + marker = FORWARD_MARKER.search(text or '') + if not marker: + return '' + match = FORWARDED_FROM.search(text, marker.end()) + return email.utils.parseaddr(match.group(1))[1].lower() if match else '' + + def retain_message(root, registry, raw, mailbox, received_at): """Retain one RFC 822 message and its attachments. Returns the delivery id.""" message = email.message_from_bytes(raw, policy=email.policy.default) @@ -114,11 +130,14 @@ def retain_message(root, registry, raw, mailbox, received_at): sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower() subject = str(message.get('Subject', '')) list_id = str(message.get('List-Id', '')) - source = reg.source_for_mail(registry, sender, subject, list_id) + text = _text(message) + forwarded_from = forwarded_sender(text) + source = reg.source_for_mail(registry, sender, subject, list_id, forwarded_from) provenance = { 'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at, 'message_id': str(message.get('Message-ID', '')), 'sender': sender, - 'subject': subject, 'list_id': list_id, 'source': source, + 'subject': subject, 'list_id': list_id, 'forwarded_from': forwarded_from, + 'source': source, 'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])], } delivery = store.record_delivery(root, item, provenance) @@ -136,7 +155,7 @@ def retain_message(root, registry, raw, mailbox, received_at): 'content_type': part.get_content_type()}) store.put_derived(root, item, { 'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')), - 'text': _text(message), 'attachments': attachments}) + 'text': text, 'attachments': attachments}) return delivery @@ -163,8 +182,9 @@ def requeue_quarantine(root, registry=None): for delivery in store.records(root, 'deliveries'): if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap': continue + text = (store.get_derived(root, delivery['item']) or {}).get('text', '') source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'), - delivery.get('list_id')) + delivery.get('list_id'), forwarded_sender(text)) if source: raw = store.get_item(root, delivery['item']) routed.append(retain_message(root, registry, raw, delivery['mailbox'], diff --git a/inbound_library/registry.py b/inbound_library/registry.py index 5c050f1..a57e522 100644 --- a/inbound_library/registry.py +++ b/inbound_library/registry.py @@ -51,7 +51,8 @@ def validate(registry): match = source.get('match', {}) if not match.get('from'): problems.append(f'{where}: email source needs match.from') - key = (match.get('from', '').lower(), match.get('subject', ''), match.get('list_id', '')) + key = (match.get('from', '').lower(), match.get('subject', ''), match.get('list_id', ''), + match.get('forwarded_from', '').lower()) if key in rules: problems.append(f'{where}: duplicate match rule with {rules[key]}') rules[key] = ident @@ -80,7 +81,7 @@ def validate(registry): return problems -def source_for_mail(registry, sender, subject='', list_id=''): +def source_for_mail(registry, sender, subject='', list_id='', forwarded_from=''): """Return the email source whose match rule fits, or None (quarantine).""" sender = (sender or '').lower() for ident, source in registry['sources'].items(): @@ -93,6 +94,9 @@ def source_for_mail(registry, sender, subject='', list_id=''): continue if match.get('list_id') and match['list_id'] != list_id: continue + if match.get('forwarded_from') and \ + match['forwarded_from'].lower() != (forwarded_from or '').lower(): + continue return ident return None diff --git a/registry/sources/grok-briefs-via-founder.json b/registry/sources/grok-briefs-via-founder.json new file mode 100644 index 0000000..4f33fec --- /dev/null +++ b/registry/sources/grok-briefs-via-founder.json @@ -0,0 +1,13 @@ +{ + "id": "grok-briefs-via-founder", + "owner_repo": "intelligence-radar", + "channel": "email", + "match": { + "from": "bernd.worsch@gmail.com", + "forwarded_from": "noreply@x.ai" + }, + "topics": ["intelligence.daily-brief", "intelligence.weekly-brief"], + "cadence": "daily", + "visibility": "shared", + "notes": "Grok automation briefs, emailed by Grok to the founder's Gmail and forwarded to library@coulomb.social. Daily and weekly runs share one sender, so the topic is not told apart here; intelligence-radar classifies on intake." +} diff --git a/tests/test_library.py b/tests/test_library.py index 08fddab..e0f160a 100644 --- a/tests/test_library.py +++ b/tests/test_library.py @@ -135,6 +135,22 @@ class Library(unittest.TestCase): self.assertEqual(notify.health(self.root)['quarantined_items'], 0) self.assertEqual(len(notify.match(self.root)), 1) + def test_forwarded_mail_routes_only_with_matching_original_sender(self): + write(self.root / 'registry' / 'sources' / 'grok-via-founder.json', { + 'id': 'grok-via-founder', 'owner_repo': 'intelligence-radar', 'channel': 'email', + 'match': {'from': 'founder@example.org', 'forwarded_from': 'noreply@x.ai'}, + 'topics': ['intelligence.daily-brief'], 'cadence': 'daily', 'visibility': 'shared'}) + grok = ('Hi\n\n---------- Forwarded message ---------\nVon: Grok \n' + 'Subject: Brief\n\nbody') + other = ('---------- Forwarded message ---------\nFrom: Someone \n\nbody') + self.drop('1.eml', mail('Founder ', 'Fwd: Brief', grok)) + self.drop('2.eml', mail('Founder ', 'Fwd: Other', other)) + first, second = intake.scan(self.root, self.mailbox()) + one = store.get_record(self.root, 'deliveries', first) + two = store.get_record(self.root, 'deliveries', second) + self.assertEqual((one['source'], one['forwarded_from']), ('grok-via-founder', 'noreply@x.ai')) + self.assertEqual((two['status'], two['forwarded_from']), ('quarantined', 'a@b.c')) + def test_live_mailbox_needs_credentials_from_environment(self): self.assertIsNone(intake.mailbox_from_config({}, environ={})) box = intake.mailbox_from_config({}, environ={'INBOUND_LIBRARY_IMAP_USER': 'u',