inbound-library/inbound_library/store.py

"""Append-only, content-addressed item store.

Original bytes live under data/items/<sha256> and are never overwritten or
deleted. Each delivery of those bytes is its own immutable record. Derived text
is rebuildable and may be replaced.
"""
import hashlib
import json
import os
import tempfile
from pathlib import Path

SCHEMA = 'inbound-library/v1'
HEX = set('0123456789abcdef')


def canonical(value):
    return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode()


def sha256(data):
    return hashlib.sha256(data).hexdigest()


def is_id(ident):
    return isinstance(ident, str) and len(ident) == 64 and set(ident) <= HEX


def atomic(path, data, immutable=False):
    """Write via a temp file. With immutable, an existing path is never replaced."""
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, name = tempfile.mkstemp(dir=path.parent, prefix='.write-')
    try:
        with os.fdopen(fd, 'wb') as stream:
            stream.write(data)
            stream.flush()
            os.fsync(stream.fileno())
        if immutable:
            try:
                os.link(name, path)
            except FileExistsError:
                if path.read_bytes() != data:
                    raise ValueError(f'refusing to overwrite {path}')
            return
        os.replace(name, path)
    finally:
        if os.path.exists(name):
            os.unlink(name)


def data_dir(root):
    return Path(root) / 'data'


def put_record(root, kind, value):
    """Store an immutable JSON record keyed by the hash of its canonical form."""
    content = canonical(value)
    ident = sha256(content)
    atomic(data_dir(root) / kind / f'{ident}.json', content + b'\n', immutable=True)
    return ident


def get_record(root, kind, ident):
    if not is_id(ident):
        raise ValueError('invalid content ID')
    value = json.loads((data_dir(root) / kind / f'{ident}.json').read_text())
    if sha256(canonical(value)) != ident:
        raise ValueError(f'integrity mismatch: {kind}/{ident}')
    return value


def records(root, kind):
    directory = data_dir(root) / kind
    if not directory.is_dir():
        return []
    return [get_record(root, kind, p.stem) for p in sorted(directory.glob('*.json'))]


def put_item(root, data):
    """Retain original bytes. Returns (item_id, newly_stored)."""
    ident = sha256(data)
    path = data_dir(root) / 'items' / ident
    existed = path.exists()
    atomic(path, data, immutable=True)
    return ident, not existed


def get_item(root, ident):
    if not is_id(ident):
        raise ValueError('invalid item ID')
    data = (data_dir(root) / 'items' / ident).read_bytes()
    if sha256(data) != ident:
        raise ValueError(f'integrity mismatch: items/{ident}')
    return data


def record_delivery(root, item, provenance):
    """Record one delivery of an item. Without a source it is quarantined."""
    if not is_id(item):
        raise ValueError('invalid item ID')
    delivery = {'schema': SCHEMA, 'item': item, **provenance}
    delivery['status'] = 'routed' if provenance.get('source') else 'quarantined'
    return put_record(root, 'deliveries', delivery)


def put_derived(root, item, derived):
    """Derived text and metadata are rebuildable, so they may be replaced."""
    atomic(data_dir(root) / 'derived' / f'{item}.json',
           canonical({'schema': SCHEMA, 'item': item, **derived}) + b'\n')


def get_derived(root, item):
    path = data_dir(root) / 'derived' / f'{item}.json'
    return json.loads(path.read_text()) if path.exists() else None


def verify(root):
    """Rehash every item and record. Returns a list of problems."""
    problems = []
    base = data_dir(root)
    items = base / 'items'
    if items.is_dir():
        for path in sorted(items.iterdir()):
            if path.name.startswith('.'):
                continue
            if not is_id(path.name) or sha256(path.read_bytes()) != path.name:
                problems.append(f'item integrity mismatch: {path.name}')
    for kind in ('deliveries', 'notices'):
        directory = base / kind
        if not directory.is_dir():
            continue
        for path in sorted(directory.glob('*.json')):
            try:
                value = get_record(root, kind, path.stem)
            except (ValueError, json.JSONDecodeError):
                problems.append(f'record integrity mismatch: {kind}/{path.stem}')
                continue
            if kind == 'deliveries' and not (items / value.get('item', '')).is_file():
                problems.append(f'delivery {path.stem} refers to missing item')
    return problems