coulomb/content-explorer
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Freeze the v0.1 contract and the local ingest connector.

Browse source 
The snapshot and manifest schemas, both example dataspaces, and the shared text renderer now pass the fixture validator and the negative contract tests. JSON, NDJSON, and Markdown frontmatter publish a snapshot only when rights, identity, and references allow it. CONTENT-WP-0001 and CONTENT-WP-0003 are marked finished against those artifacts. CONTENT-WP-0000-T04 stays open. SCOPE.md and the 19:05 scope-against-intent note record the stop.

Assistant: grok
Assistant-Session: 01a130f1-abfb-7d71-9cd6-a8b75ec7f95a
This commit is contained in:
tegwick 2026-10-12 19:07:32 +02:00
parent 2b1706559b
commit 072a7fa550
30 changed files with 1896 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

371
content_explorer/ingest.py Normal file
View file

@ -0,0 +1,371 @@
"""Build a v0.1 snapshot from JSON, NDJSON, or Markdown frontmatter.
Connectors stop at a snapshot and a report. They do not render.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
from pathlib import Path
from typing import Any
import yaml
CONNECTOR_VERSION = "0.1.0"
ALLOWED_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "MIT"})
def _digest(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def snapshot_hash(snapshot: dict) -> str:
encoded = json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode("utf-8")
return _digest(encoded)
def load_json_records(path: Path) -> tuple[list[dict], str]:
raw = path.read_bytes()
document = json.loads(raw.decode("utf-8"))
if isinstance(document, dict) and "records" in document:
records = document["records"]
elif isinstance(document, list):
records = document
else:
raise ValueError(f"{path} is not a record list")
return records, _digest(raw)
def load_ndjson_records(path: Path) -> tuple[list[dict], str]:
raw = path.read_bytes()
records = []
for line in raw.decode("utf-8").splitlines():
if line.strip():
records.append(json.loads(line))
return records, _digest(raw)
def load_markdown_records(directory: Path) -> tuple[list[dict], str]:
parts: list[bytes] = []
records: list[dict] = []
for path in sorted(directory.glob("*.md")):
raw = path.read_bytes()
parts.append(raw)
text = raw.decode("utf-8")
if not text.startswith("---\n"):
raise ValueError(f"{path.name} has no frontmatter")
_, frontmatter, body = text.split("---", 2)
record = yaml.safe_load(frontmatter) or {}
if not isinstance(record, dict):
raise ValueError(f"{path.name} frontmatter is not a mapping")
if body.strip() and not record.get("summary"):
record["summary"] = body.strip()
records.append(record)
return records, _digest(b"".join(parts))
def _entity_id(dataspace: str, kind: str, key: str) -> str:
return f"urn:ce:{dataspace}:{kind}:{key}"
def _unpublished(report: dict) -> dict[str, Any]:
return {"snapshot": None, "manifest": None, "report": report}
def build_manifest(
*,
dataspace_id: str,
heading: str,
kinds: list[dict],
relation_labels: dict[str, dict[str, str]],
predicates: list[str],
fact_policy: str = "show-all",
acyclic_predicates: list[str] | None = None,
) -> tuple[dict[str, Any] | None, list[str]]:
"""Build a v0.1 manifest. Missing relation labels are returned and not invented."""
missing = sorted({predicate for predicate in predicates if predicate not in relation_labels})
if missing:
return None, missing
kind_ids = [kind["id"] for kind in kinds]
labels = {
predicate: {
"forward": relation_labels[predicate]["forward"],
"inverse": relation_labels[predicate]["inverse"],
}
for predicate in sorted(set(predicates))
}
manifest = {
"schemaVersion": "0.1",
"id": dataspace_id,
"factPolicy": fact_policy,
"home": {"heading": heading, "featuredKinds": kind_ids},
"navigation": [{"label": "Browse", "kinds": kind_ids}],
"kindViews": {
kind["id"]: {"columns": ["title"], "facets": ["tags"]}
for kind in kinds
},
"panels": ["summary", "relations", "evidence", "claims"],
"relationLabels": labels,
"acyclicPredicates": list(acyclic_predicates or []),
}
return manifest, []
def human_import_report(report: dict) -> str:
"""Provenance and rights summary a person can read next to the snapshot."""
source = report.get("source") or {}
lines = [
"# Import report",
"",
f"Connector version: {report.get('connectorVersion', '')}",
f"Source id: {source.get('id', '')}",
f"Source URI: {source.get('uri', '')}",
f"License: {source.get('license', '')}",
f"Attribution: {source.get('attribution') or '(none)'}",
f"Source digest: {report.get('sourceDigest') or '(none)'}",
f"Content hash: {report.get('contentHash') or '(not published)'}",
f"Published: {str(bool(report.get('published'))).lower()}",
f"Candidates: {report.get('candidates', 0)}",
f"Accepted: {report.get('accepted', 0)}",
"",
"## Tombstones",
]
tombstones = report.get("tombstoned") or []
lines.extend(f"- {key}" for key in tombstones)
if not tombstones:
lines.append("- (none)")
lines.extend(["", "## Quarantine"])
quarantined = report.get("quarantined") or []
if not quarantined:
lines.append("- (none)")
for item in quarantined:
lines.append(f"- {json.dumps(item, sort_keys=True)}")
lines.extend(["", "## Dangling references"])
dangling = report.get("dangling") or []
if not dangling:
lines.append("- (none)")
for item in dangling:
lines.append(f"- {item.get('from')} -{item.get('predicate')}-> {item.get('to')}")
lines.extend(["", "## Identity conflicts"])
conflicts = report.get("conflicts") or []
if not conflicts:
lines.append("- (none)")
for item in conflicts:
lines.append(f"- {item.get('key')}: {item.get('reason')}")
lines.append("")
return "\n".join(lines)
def project_records(
records: list[dict],
*,
dataspace_id: str,
title: str,
description: str,
source: dict,
kinds: list[dict],
observed_at: str,
source_digest: str = "",
relation_labels: dict[str, dict[str, str]] | None = None,
fact_policy: str = "show-all",
acyclic_predicates: list[str] | None = None,
) -> dict[str, Any]:
"""Return a report. `snapshot` is present only when publication is allowed.
Live keys are collected first. Relations resolve in a second pass against
that set. Duplicate keys and a live key that is also tombstoned are conflicts.
"""
license_name = str(source.get("license") or "")
report: dict[str, Any] = {
"connectorVersion": CONNECTOR_VERSION,
"candidates": len(records),
"accepted": 0,
"tombstoned": [],
"quarantined": [],
"dangling": [],
"conflicts": [],
"published": False,
"sourceDigest": source_digest,
"source": {
"id": source.get("id", ""),
"uri": source.get("uri", ""),
"title": source.get("title", ""),
"license": license_name,
"attribution": source.get("attribution") or "",
},
}
if license_name not in ALLOWED_LICENSES:
report["quarantined"].append({
"reason": "rights",
"license": license_name or "(missing)",
})
return _unpublished(report)
if license_name == "CC-BY-4.0" and not str(source.get("attribution") or "").strip():
report["quarantined"].append({
"reason": "rights",
"license": license_name,
"detail": "missing-attribution",
})
return _unpublished(report)
kind_ids = {kind["id"] for kind in kinds}
tombstoned_keys = set()
for record in records:
if record.get("deleted"):
report["tombstoned"].append(record.get("key"))
tombstoned_keys.add(f"{record.get('kind')}:{record.get('key')}")
grouped: dict[str, list[dict]] = {}
for record in records:
if record.get("deleted"):
continue
kind = record["kind"]
key = record["key"]
qualified = f"{kind}:{key}"
if kind not in kind_ids:
report["quarantined"].append({"reason": "unknown-kind", "key": key, "kind": kind})
continue
if qualified in tombstoned_keys:
report["conflicts"].append({"key": qualified, "reason": "tombstone-and-live"})
continue
grouped.setdefault(qualified, []).append(record)
accepted: dict[str, dict] = {}
for qualified, group in grouped.items():
if len(group) > 1:
report["conflicts"].append({
"key": qualified,
"reason": "duplicate-identity",
"count": len(group),
})
continue
accepted[qualified] = group[0]
entities = []
relations = []
claims = []
predicates: list[str] = []
# Second pass: links resolve only against keys accepted in the first pass.
for qualified, record in accepted.items():
kind, key = qualified.split(":", 1)
entity_id = _entity_id(dataspace_id, kind, key)
for link in record.get("relations") or []:
target = link.get("to")
predicate = link.get("predicate")
if target not in accepted:
report["dangling"].append({"from": qualified, "to": target, "predicate": predicate})
claim_id = f"claim:{kind}:{key}:tldr"
entities.append({
"id": entity_id,
"kindId": kind,
"title": record["title"],
"tldr": record["tldr"],
"summary": record.get("summary") or record["tldr"],
"tags": list(record.get("tags") or []),
"sourceRefs": [source["id"]],
"claimRefs": [claim_id],
"asOf": observed_at,
})
claims.append({
"id": claim_id,
"subjectId": entity_id,
"key": "tldr",
"value": record["tldr"],
"status": "asserted",
"evidenceRefs": [source["id"]],
"observedAt": observed_at,
})
for index, link in enumerate(record.get("relations") or [], start=1):
target = link["to"]
predicate = link["predicate"]
predicates.append(predicate)
target_kind, target_key = target.split(":", 1)
relations.append({
"id": f"{kind}-{key}-{index}",
"predicate": predicate,
"from": entity_id,
"to": _entity_id(dataspace_id, target_kind, target_key),
"sourceRefs": [source["id"]],
"asOf": observed_at,
})
blocked = (
report["dangling"]
or report["conflicts"]
or any(item["reason"] == "unknown-kind" for item in report["quarantined"])
)
if blocked:
return _unpublished(report)
manifest, missing_labels = build_manifest(
dataspace_id=dataspace_id,
heading=title,
kinds=kinds,
relation_labels=relation_labels or {},
predicates=predicates,
fact_policy=fact_policy,
acyclic_predicates=acyclic_predicates,
)
if missing_labels:
report["quarantined"].append({
"reason": "missing-relation-label",
"predicates": missing_labels,
})
return _unpublished(report)
entities.sort(key=lambda item: item["id"])
relations.sort(key=lambda item: item["id"])
claims.sort(key=lambda item: item["id"])
snapshot = {
"schemaVersion": "0.1",
"dataspace": {"id": dataspace_id, "title": title, "description": description},
"kinds": kinds,
"entities": entities,
"relations": relations,
"sources": [source],
"claims": claims,
}
for claim in claims:
if not claim["evidenceRefs"]:
report["quarantined"].append({"reason": "missing-evidence", "claim": claim["id"]})
return _unpublished(report)
report["accepted"] = len(entities)
report["published"] = True
report["contentHash"] = snapshot_hash(snapshot)
return {"snapshot": snapshot, "manifest": manifest, "report": report}
def publish_directory(destination: Path, result: dict[str, Any]) -> None:
"""Replace `destination` only after the snapshot, manifest, and both reports exist."""
if not result["report"]["published"] or result["snapshot"] is None or result.get("manifest") is None:
raise RuntimeError("refusing to publish a quarantined or dangling import")
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.parent / f".{destination.name}.incoming"
if temporary.exists():
shutil.rmtree(temporary)
temporary.mkdir()
(temporary / "snapshot.json").write_text(
json.dumps(result["snapshot"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "manifest.json").write_text(
json.dumps(result["manifest"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "import-report.json").write_text(
json.dumps(result["report"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "import-report.md").write_text(
human_import_report(result["report"]),
encoding="utf-8",
)
backup = destination.parent / f".{destination.name}.backup"
if backup.exists():
shutil.rmtree(backup)
if destination.exists():
os.rename(destination, backup)
os.rename(temporary, destination)