Route forwarded mail by its original sender; register Grok briefs via founder.
Adds forwarded_from (original sender from the forwarded header block, English/German) to delivery provenance and source match rules, so only founder forwards of noreply@x.ai mail route as Grok briefs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 1170398@bnt-lap001 Assistant-Session: 622cd705-cb31-4322-ba08-dd42e44fa583
This commit is contained in:
parent
96e439495c
commit
65887ef637
5 changed files with 61 additions and 7 deletions
|
|
@ -41,6 +41,7 @@ One record per delivery of an item.
|
||||||
| `received_at` | IMAP INTERNALDATE (UTC ISO 8601) |
|
| `received_at` | IMAP INTERNALDATE (UTC ISO 8601) |
|
||||||
| `mailbox` | `<user>/<folder>`; for `imap` only |
|
| `mailbox` | `<user>/<folder>`; for `imap` only |
|
||||||
| `message_id`, `sender`, `subject`, `list_id` | for `imap` only; `sender` is lowercased |
|
| `message_id`, `sender`, `subject`, `list_id` | for `imap` only; `sender` is lowercased |
|
||||||
|
| `forwarded_from` | for `imap`: the original sender from the first forwarded header block in the body (`From:`/`Von:` after a forward marker), or `''` |
|
||||||
| `authentication_results` | raw headers, recorded as provenance, not as trust |
|
| `authentication_results` | raw headers, recorded as provenance, not as trust |
|
||||||
| `parent_item`, `filename`, `content_type` | for `attachment` only |
|
| `parent_item`, `filename`, `content_type` | for `attachment` only |
|
||||||
|
|
||||||
|
|
@ -69,7 +70,7 @@ subscriptions may only use registered topics.
|
||||||
| `id` | the file name without `.json` |
|
| `id` | the file name without `.json` |
|
||||||
| `owner_repo` | the repository that registered it |
|
| `owner_repo` | the repository that registered it |
|
||||||
| `channel` | `email` or `url` |
|
| `channel` | `email` or `url` |
|
||||||
| `match` | for `email`: `from` (required), `subject` (substring), `list_id` |
|
| `match` | for `email`: `from` (required), `subject` (substring), `list_id`, `forwarded_from` (the original sender of a forwarded mail) |
|
||||||
| `url` | for `url` |
|
| `url` | for `url` |
|
||||||
| `topics` | list of registered topics |
|
| `topics` | list of registered topics |
|
||||||
| `cadence` | `daily`, `weekly` or `irregular`; daily and weekly email sources are watched for silence (36 h or 9 days) |
|
| `cadence` | `daily`, `weekly` or `irregular`; daily and weekly email sources are watched for silence (36 h or 9 days) |
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,22 @@ def _text(message):
|
||||||
return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip()
|
return html.unescape(re.sub(r'(?s)<[^>]+>', '', stripped)).strip()
|
||||||
|
|
||||||
|
|
||||||
|
# Header line of a forwarded message, as Gmail and common clients write it in the
|
||||||
|
# body, in English or German.
|
||||||
|
FORWARDED_FROM = re.compile(r'^\s*(?:From|Von)\s*:\s*(.+)$', re.MULTILINE)
|
||||||
|
FORWARD_MARKER = re.compile(r'-{3,}\s*(?:Forwarded message|Weitergeleitete Nachricht)\s*-{3,}',
|
||||||
|
re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
def forwarded_sender(text):
|
||||||
|
"""Original sender of a forwarded mail, from the first header block after the marker."""
|
||||||
|
marker = FORWARD_MARKER.search(text or '')
|
||||||
|
if not marker:
|
||||||
|
return ''
|
||||||
|
match = FORWARDED_FROM.search(text, marker.end())
|
||||||
|
return email.utils.parseaddr(match.group(1))[1].lower() if match else ''
|
||||||
|
|
||||||
|
|
||||||
def retain_message(root, registry, raw, mailbox, received_at):
|
def retain_message(root, registry, raw, mailbox, received_at):
|
||||||
"""Retain one RFC 822 message and its attachments. Returns the delivery id."""
|
"""Retain one RFC 822 message and its attachments. Returns the delivery id."""
|
||||||
message = email.message_from_bytes(raw, policy=email.policy.default)
|
message = email.message_from_bytes(raw, policy=email.policy.default)
|
||||||
|
|
@ -114,11 +130,14 @@ def retain_message(root, registry, raw, mailbox, received_at):
|
||||||
sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower()
|
sender = email.utils.parseaddr(str(message.get('From', '')))[1].lower()
|
||||||
subject = str(message.get('Subject', ''))
|
subject = str(message.get('Subject', ''))
|
||||||
list_id = str(message.get('List-Id', ''))
|
list_id = str(message.get('List-Id', ''))
|
||||||
source = reg.source_for_mail(registry, sender, subject, list_id)
|
text = _text(message)
|
||||||
|
forwarded_from = forwarded_sender(text)
|
||||||
|
source = reg.source_for_mail(registry, sender, subject, list_id, forwarded_from)
|
||||||
provenance = {
|
provenance = {
|
||||||
'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at,
|
'channel': 'imap', 'mailbox': mailbox, 'received_at': received_at,
|
||||||
'message_id': str(message.get('Message-ID', '')), 'sender': sender,
|
'message_id': str(message.get('Message-ID', '')), 'sender': sender,
|
||||||
'subject': subject, 'list_id': list_id, 'source': source,
|
'subject': subject, 'list_id': list_id, 'forwarded_from': forwarded_from,
|
||||||
|
'source': source,
|
||||||
'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])],
|
'authentication_results': [str(h) for h in message.get_all('Authentication-Results', [])],
|
||||||
}
|
}
|
||||||
delivery = store.record_delivery(root, item, provenance)
|
delivery = store.record_delivery(root, item, provenance)
|
||||||
|
|
@ -136,7 +155,7 @@ def retain_message(root, registry, raw, mailbox, received_at):
|
||||||
'content_type': part.get_content_type()})
|
'content_type': part.get_content_type()})
|
||||||
store.put_derived(root, item, {
|
store.put_derived(root, item, {
|
||||||
'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')),
|
'subject': subject, 'sender': sender, 'date': str(message.get('Date', '')),
|
||||||
'text': _text(message), 'attachments': attachments})
|
'text': text, 'attachments': attachments})
|
||||||
return delivery
|
return delivery
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -163,8 +182,9 @@ def requeue_quarantine(root, registry=None):
|
||||||
for delivery in store.records(root, 'deliveries'):
|
for delivery in store.records(root, 'deliveries'):
|
||||||
if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap':
|
if delivery['status'] != 'quarantined' or delivery.get('channel') != 'imap':
|
||||||
continue
|
continue
|
||||||
|
text = (store.get_derived(root, delivery['item']) or {}).get('text', '')
|
||||||
source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'),
|
source = reg.source_for_mail(registry, delivery.get('sender'), delivery.get('subject'),
|
||||||
delivery.get('list_id'))
|
delivery.get('list_id'), forwarded_sender(text))
|
||||||
if source:
|
if source:
|
||||||
raw = store.get_item(root, delivery['item'])
|
raw = store.get_item(root, delivery['item'])
|
||||||
routed.append(retain_message(root, registry, raw, delivery['mailbox'],
|
routed.append(retain_message(root, registry, raw, delivery['mailbox'],
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,8 @@ def validate(registry):
|
||||||
match = source.get('match', {})
|
match = source.get('match', {})
|
||||||
if not match.get('from'):
|
if not match.get('from'):
|
||||||
problems.append(f'{where}: email source needs match.from')
|
problems.append(f'{where}: email source needs match.from')
|
||||||
key = (match.get('from', '').lower(), match.get('subject', ''), match.get('list_id', ''))
|
key = (match.get('from', '').lower(), match.get('subject', ''), match.get('list_id', ''),
|
||||||
|
match.get('forwarded_from', '').lower())
|
||||||
if key in rules:
|
if key in rules:
|
||||||
problems.append(f'{where}: duplicate match rule with {rules[key]}')
|
problems.append(f'{where}: duplicate match rule with {rules[key]}')
|
||||||
rules[key] = ident
|
rules[key] = ident
|
||||||
|
|
@ -80,7 +81,7 @@ def validate(registry):
|
||||||
return problems
|
return problems
|
||||||
|
|
||||||
|
|
||||||
def source_for_mail(registry, sender, subject='', list_id=''):
|
def source_for_mail(registry, sender, subject='', list_id='', forwarded_from=''):
|
||||||
"""Return the email source whose match rule fits, or None (quarantine)."""
|
"""Return the email source whose match rule fits, or None (quarantine)."""
|
||||||
sender = (sender or '').lower()
|
sender = (sender or '').lower()
|
||||||
for ident, source in registry['sources'].items():
|
for ident, source in registry['sources'].items():
|
||||||
|
|
@ -93,6 +94,9 @@ def source_for_mail(registry, sender, subject='', list_id=''):
|
||||||
continue
|
continue
|
||||||
if match.get('list_id') and match['list_id'] != list_id:
|
if match.get('list_id') and match['list_id'] != list_id:
|
||||||
continue
|
continue
|
||||||
|
if match.get('forwarded_from') and \
|
||||||
|
match['forwarded_from'].lower() != (forwarded_from or '').lower():
|
||||||
|
continue
|
||||||
return ident
|
return ident
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
13
registry/sources/grok-briefs-via-founder.json
Normal file
13
registry/sources/grok-briefs-via-founder.json
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
{
|
||||||
|
"id": "grok-briefs-via-founder",
|
||||||
|
"owner_repo": "intelligence-radar",
|
||||||
|
"channel": "email",
|
||||||
|
"match": {
|
||||||
|
"from": "bernd.worsch@gmail.com",
|
||||||
|
"forwarded_from": "noreply@x.ai"
|
||||||
|
},
|
||||||
|
"topics": ["intelligence.daily-brief", "intelligence.weekly-brief"],
|
||||||
|
"cadence": "daily",
|
||||||
|
"visibility": "shared",
|
||||||
|
"notes": "Grok automation briefs, emailed by Grok to the founder's Gmail and forwarded to library@coulomb.social. Daily and weekly runs share one sender, so the topic is not told apart here; intelligence-radar classifies on intake."
|
||||||
|
}
|
||||||
|
|
@ -135,6 +135,22 @@ class Library(unittest.TestCase):
|
||||||
self.assertEqual(notify.health(self.root)['quarantined_items'], 0)
|
self.assertEqual(notify.health(self.root)['quarantined_items'], 0)
|
||||||
self.assertEqual(len(notify.match(self.root)), 1)
|
self.assertEqual(len(notify.match(self.root)), 1)
|
||||||
|
|
||||||
|
def test_forwarded_mail_routes_only_with_matching_original_sender(self):
|
||||||
|
write(self.root / 'registry' / 'sources' / 'grok-via-founder.json', {
|
||||||
|
'id': 'grok-via-founder', 'owner_repo': 'intelligence-radar', 'channel': 'email',
|
||||||
|
'match': {'from': 'founder@example.org', 'forwarded_from': 'noreply@x.ai'},
|
||||||
|
'topics': ['intelligence.daily-brief'], 'cadence': 'daily', 'visibility': 'shared'})
|
||||||
|
grok = ('Hi\n\n---------- Forwarded message ---------\nVon: Grok <noreply@x.ai>\n'
|
||||||
|
'Subject: Brief\n\nbody')
|
||||||
|
other = ('---------- Forwarded message ---------\nFrom: Someone <a@b.c>\n\nbody')
|
||||||
|
self.drop('1.eml', mail('Founder <founder@example.org>', 'Fwd: Brief', grok))
|
||||||
|
self.drop('2.eml', mail('Founder <founder@example.org>', 'Fwd: Other', other))
|
||||||
|
first, second = intake.scan(self.root, self.mailbox())
|
||||||
|
one = store.get_record(self.root, 'deliveries', first)
|
||||||
|
two = store.get_record(self.root, 'deliveries', second)
|
||||||
|
self.assertEqual((one['source'], one['forwarded_from']), ('grok-via-founder', 'noreply@x.ai'))
|
||||||
|
self.assertEqual((two['status'], two['forwarded_from']), ('quarantined', 'a@b.c'))
|
||||||
|
|
||||||
def test_live_mailbox_needs_credentials_from_environment(self):
|
def test_live_mailbox_needs_credentials_from_environment(self):
|
||||||
self.assertIsNone(intake.mailbox_from_config({}, environ={}))
|
self.assertIsNone(intake.mailbox_from_config({}, environ={}))
|
||||||
box = intake.mailbox_from_config({}, environ={'INBOUND_LIBRARY_IMAP_USER': 'u',
|
box = intake.mailbox_from_config({}, environ={'INBOUND_LIBRARY_IMAP_USER': 'u',
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue