tegwick
29ceb77938
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a09c77-b1a9-7eb3-b501-6cb24b458c97
90 lines
4.1 KiB
Python
90 lines
4.1 KiB
Python
"""Bounded public HTTP capture; fetched pages are evidence, not verified claims."""
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
from urllib.parse import urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
from .store import all_records, atomic, put
|
|
|
|
MAX_BYTES = 5 * 1024 * 1024
|
|
|
|
|
|
def capture(root, source, opener=urlopen, now=None):
|
|
url = source['url']
|
|
parsed = urlparse(url)
|
|
if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ValueError('sources require public HTTPS URLs without credentials')
|
|
if not source.get('id') or not source.get('publisher'):
|
|
raise ValueError('source id and publisher required')
|
|
timestamp = (now or datetime.now(timezone.utc)).isoformat()
|
|
attempts = 0
|
|
for attempt in range(3):
|
|
attempts += 1
|
|
try:
|
|
request = Request(url, headers={'User-Agent': 'IntelligenceRadar/0.2 (research source monitor)',
|
|
'Accept': 'text/html,application/json,application/xml,text/plain'})
|
|
with opener(request, timeout=20) as response:
|
|
final_url = response.geturl()
|
|
if urlparse(final_url).scheme != 'https':
|
|
raise ValueError('source redirected away from HTTPS')
|
|
body = response.read(MAX_BYTES + 1)
|
|
media_type = response.headers.get('Content-Type', 'application/octet-stream')
|
|
break
|
|
except OSError:
|
|
if attempt == 2:
|
|
raise
|
|
time.sleep(0.25 * (2 ** attempt))
|
|
if len(body) > MAX_BYTES:
|
|
raise ValueError('source exceeds 5 MiB capture limit')
|
|
content_hash = hashlib.sha256(body).hexdigest()
|
|
artifact = Path(root) / 'data/artifacts' / content_hash
|
|
atomic(artifact, body, immutable=True)
|
|
previous = [r for _, r in all_records(root, 'captures') if r['source_id'] == source['id']]
|
|
previous.sort(key=lambda r: r['retrieved_at'])
|
|
changed = not previous or previous[-1]['content_sha256'] != content_hash
|
|
receipt = {'schema_version': 1, 'source_id': source['id'], 'publisher': source['publisher'],
|
|
'url': url, 'final_url': final_url, 'retrieved_at': timestamp,
|
|
'content_sha256': content_hash, 'media_type': media_type,
|
|
'bytes': len(body), 'changed': changed, 'attempts': attempts,
|
|
'status': 'captured_unreviewed'}
|
|
ident = put(root, 'captures', receipt)
|
|
if changed:
|
|
from .__main__ import record
|
|
record(root, {'schema_version': 1, 'title': f"Source update: {source['id']}",
|
|
'claim': 'Source content was captured or changed. Extracted claims require editorial review.',
|
|
'source_url': final_url, 'source_kind': 'primary',
|
|
'content_sha256': content_hash, 'observed_at': timestamp,
|
|
'event_date': timestamp[:10], 'status': 'reported',
|
|
'observer': 'intelligence-radar/capture-v0.2', 'notes': f'Capture ID: {ident}'})
|
|
return ident, receipt
|
|
|
|
|
|
def ingest(root, registry):
|
|
sources = registry['sources']
|
|
ids = [s['id'] for s in sources]
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError('duplicate source IDs')
|
|
results = []
|
|
for source in sources:
|
|
try:
|
|
ident, receipt = capture(root, source)
|
|
results.append({'source_id': source['id'], 'capture_id': ident,
|
|
'ok': True, 'changed': receipt['changed']})
|
|
except (ValueError, OSError) as exc:
|
|
results.append({'source_id': source['id'], 'ok': False, 'error': str(exc)})
|
|
run = {'observed_at': datetime.now(timezone.utc).isoformat(), 'sources': results,
|
|
'ok': all(r['ok'] for r in results)}
|
|
put(root, 'ingestion-runs', run)
|
|
return run
|
|
|
|
|
|
def check_artifacts(root):
|
|
captures = all_records(root, 'captures')
|
|
for _, value in captures:
|
|
path = Path(root) / 'data/artifacts' / value['content_sha256']
|
|
if hashlib.sha256(path.read_bytes()).hexdigest() != value['content_sha256']:
|
|
raise ValueError(f'artifact integrity mismatch: {path}')
|
|
return len(captures)
|