coulomb/content-explorer
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Freeze the v0.1 contract and the local ingest connector.

Browse source 
The snapshot and manifest schemas, both example dataspaces, and the shared text renderer now pass the fixture validator and the negative contract tests. JSON, NDJSON, and Markdown frontmatter publish a snapshot only when rights, identity, and references allow it. CONTENT-WP-0001 and CONTENT-WP-0003 are marked finished against those artifacts. CONTENT-WP-0000-T04 stays open. SCOPE.md and the 19:05 scope-against-intent note record the stop.

Assistant: grok
Assistant-Session: 01a130f1-abfb-7d71-9cd6-a8b75ec7f95a
This commit is contained in:
tegwick 2026-10-12 19:07:32 +02:00
parent 2b1706559b
commit 072a7fa550
30 changed files with 1896 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

3
content_explorer/__init__.py Normal file
View file

@ -0,0 +1,3 @@
"""Domain-independent snapshot contracts and rendering."""
__version__ = "0.1.0"

236
content_explorer/contract.py Normal file
View file

@ -0,0 +1,236 @@
"""Validate a v0.1 snapshot against its manifest.
Publication checks live here. Callers pass documents they already loaded.
`validate_dataspace` is the file entry point used by `scripts/validate_fixtures.py`.
"""
from __future__ import annotations
import json
from collections import defaultdict
from pathlib import Path
from typing import Any
import jsonschema
ROOT = Path(__file__).resolve().parent.parent
SNAPSHOT_SCHEMA_PATH = ROOT / "contracts" / "snapshot.schema.json"
MANIFEST_SCHEMA_PATH = ROOT / "contracts" / "manifest.schema.json"
SUPPORTED_VERSIONS = frozenset({"0.1"})
ENTITY_ID_PREFIX = "urn:ce:"
def load_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def _validator(schema_path: Path) -> jsonschema.Draft202012Validator:
schema = load_json(schema_path)
return jsonschema.Draft202012Validator(
schema,
format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER,
)
def schema_errors(document: dict, schema_path: Path) -> list[str]:
validator = _validator(schema_path)
found = sorted(validator.iter_errors(document), key=lambda err: list(err.path))
messages = []
for err in found:
where = "/".join(str(part) for part in err.path) or "(root)"
messages.append(f"schema {where}: {err.message}")
return messages
def version_error(document: dict, label: str) -> str | None:
version = document.get("schemaVersion")
if version in SUPPORTED_VERSIONS:
return None
return (
f"{label} schemaVersion {version!r} is not supported; "
f"expected one of {', '.join(sorted(SUPPORTED_VERSIONS))}"
)
def resolve_claims(claims: list[dict], key: str, policy: str) -> dict[str, Any]:
"""Choose what a display field may show for one subject.
`missing` means no claim exists. `unknown` means a claim records that the
value is not known. Differing asserted values are a contradiction. The
manifest policy `show-all` keeps every asserted value, `withhold` keeps
none, and `preferred` keeps the claim marked preferred or withholds when
none is marked.
"""
relevant = [claim for claim in claims if claim.get("key") == key]
if not relevant:
return {"state": "missing", "values": []}
asserted = [claim for claim in relevant if claim.get("status") == "asserted"]
values = []
for claim in asserted:
value = claim.get("value")
if value not in values:
values.append(value)
if len(values) > 1:
if policy == "show-all":
return {"state": "contradicted", "values": values}
if policy == "preferred":
chosen = [claim.get("value") for claim in asserted if claim.get("preferred") is True]
if len(chosen) == 1:
return {"state": "present", "values": chosen}
return {"state": "contradicted", "values": []}
return {"state": "contradicted", "values": []}
if values:
return {"state": "present", "values": values}
if any(claim.get("status") == "unknown" for claim in relevant):
return {"state": "unknown", "values": []}
if any(claim.get("status") == "withheld" for claim in relevant):
return {"state": "withheld", "values": []}
return {"state": "missing", "values": []}
def _cycles(snapshot: dict, predicates: set[str]) -> list[str]:
if not predicates:
return []
adjacency: dict[str, list[tuple[str, str]]] = defaultdict(list)
for relation in snapshot.get("relations", []):
if relation.get("predicate") in predicates:
adjacency[relation["from"]].append((relation["to"], relation["id"]))
color: dict[str, int] = {}
errors: list[str] = []
def walk(node: str) -> None:
color[node] = 1
for nxt, relation_id in adjacency.get(node, []):
state = color.get(nxt, 0)
if state == 1:
errors.append(
f"taxonomic cycle via {relation_id}: {node} -> {nxt}"
)
elif state == 0:
walk(nxt)
color[node] = 2
for start in list(adjacency):
if color.get(start, 0) == 0:
walk(start)
return errors
def graph_errors(snapshot: dict, manifest: dict) -> list[str]:
errors: list[str] = []
version = version_error(snapshot, "snapshot")
if version:
errors.append(version)
manifest_version = version_error(manifest, "manifest")
if manifest_version:
errors.append(manifest_version)
kinds = [item.get("id") for item in snapshot.get("kinds", [])]
if len(kinds) != len(set(kinds)):
errors.append("duplicate kind id")
kind_ids = set(kinds)
entity_ids = [item.get("id") for item in snapshot.get("entities", [])]
if len(entity_ids) != len(set(entity_ids)):
errors.append("duplicate entity id")
entities = {item.get("id"): item for item in snapshot.get("entities", []) if item.get("id")}
sources = {item.get("id") for item in snapshot.get("sources", [])}
claim_ids = [item.get("id") for item in snapshot.get("claims", [])]
if len(claim_ids) != len(set(claim_ids)):
errors.append("duplicate claim id")
dataspace_id = (snapshot.get("dataspace") or {}).get("id")
if manifest.get("id") != dataspace_id:
errors.append(
f"manifest id {manifest.get('id')!r} does not match dataspace {dataspace_id!r}"
)
labels = manifest.get("relationLabels") or {}
views = manifest.get("kindViews") or {}
for kind_id in kind_ids:
if kind_id not in views:
errors.append(f"kind {kind_id} has no manifest kindView")
for featured in (manifest.get("home") or {}).get("featuredKinds", []):
if featured not in kind_ids:
errors.append(f"featured kind {featured} is not in the snapshot")
for entity in snapshot.get("entities", []):
entity_id = entity.get("id", "")
if not str(entity_id).startswith(ENTITY_ID_PREFIX):
errors.append(f"entity id {entity_id} is not a urn:ce identifier")
if entity.get("kindId") not in kind_ids:
errors.append(f"{entity_id}: undeclared kind {entity.get('kindId')}")
if not entity.get("sourceRefs"):
errors.append(f"{entity_id}: missing source")
for source_id in entity.get("sourceRefs", []):
if source_id not in sources:
errors.append(f"{entity_id}: missing source {source_id}")
for claim_id in entity.get("claimRefs", []):
if claim_id not in claim_ids:
errors.append(f"{entity_id}: missing claim {claim_id}")
known_entities = set(entities)
for relation in snapshot.get("relations", []):
relation_id = relation.get("id")
predicate = relation.get("predicate")
if predicate not in labels:
errors.append(f"relation {relation_id}: predicate {predicate} has no label")
for field in ("from", "to"):
target = relation.get(field)
if target not in known_entities:
errors.append(f"relation {relation_id}: dangling {field} {target}")
refs = relation.get("sourceRefs") or []
if not refs:
errors.append(f"relation {relation_id}: missing source")
for source_id in refs:
if source_id not in sources:
errors.append(f"relation {relation_id}: missing source {source_id}")
for claim in snapshot.get("claims", []):
claim_id = claim.get("id")
subject = claim.get("subjectId")
if subject not in known_entities:
errors.append(f"claim {claim_id}: dangling subject {subject}")
evidence = claim.get("evidenceRefs") or []
if not evidence:
errors.append(f"claim {claim_id}: missing source")
for source_id in evidence:
if source_id not in sources:
errors.append(f"claim {claim_id}: missing source {source_id}")
acyclic = set(manifest.get("acyclicPredicates") or [])
errors.extend(_cycles(snapshot, acyclic))
return errors
def size_errors(snapshot: dict, *, min_kinds: int = 3, min_entities: int = 8) -> list[str]:
errors = []
kind_count = len(snapshot.get("kinds") or [])
entity_count = len(snapshot.get("entities") or [])
if kind_count < min_kinds:
errors.append(f"needs at least {min_kinds} kinds, found {kind_count}")
if entity_count < min_entities:
errors.append(f"needs at least {min_entities} entities, found {entity_count}")
return errors
def validate_documents(
snapshot: dict,
manifest: dict,
*,
require_size: bool = False,
) -> list[str]:
errors = schema_errors(snapshot, SNAPSHOT_SCHEMA_PATH)
errors.extend(schema_errors(manifest, MANIFEST_SCHEMA_PATH))
errors.extend(graph_errors(snapshot, manifest))
if require_size:
errors.extend(size_errors(snapshot))
return errors
def validate_dataspace(directory: Path, *, require_size: bool = False) -> list[str]:
snapshot_path = directory / "snapshot.json"
manifest_path = directory / "manifest.json"
snapshot = load_json(snapshot_path)
manifest = load_json(manifest_path)
errors = validate_documents(snapshot, manifest, require_size=require_size)
return [f"{directory.name}: {message}" for message in errors]

371
content_explorer/ingest.py Normal file
View file

@ -0,0 +1,371 @@
"""Build a v0.1 snapshot from JSON, NDJSON, or Markdown frontmatter.
Connectors stop at a snapshot and a report. They do not render.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
from pathlib import Path
from typing import Any
import yaml
CONNECTOR_VERSION = "0.1.0"
ALLOWED_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "MIT"})
def _digest(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def snapshot_hash(snapshot: dict) -> str:
encoded = json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode("utf-8")
return _digest(encoded)
def load_json_records(path: Path) -> tuple[list[dict], str]:
raw = path.read_bytes()
document = json.loads(raw.decode("utf-8"))
if isinstance(document, dict) and "records" in document:
records = document["records"]
elif isinstance(document, list):
records = document
else:
raise ValueError(f"{path} is not a record list")
return records, _digest(raw)
def load_ndjson_records(path: Path) -> tuple[list[dict], str]:
raw = path.read_bytes()
records = []
for line in raw.decode("utf-8").splitlines():
if line.strip():
records.append(json.loads(line))
return records, _digest(raw)
def load_markdown_records(directory: Path) -> tuple[list[dict], str]:
parts: list[bytes] = []
records: list[dict] = []
for path in sorted(directory.glob("*.md")):
raw = path.read_bytes()
parts.append(raw)
text = raw.decode("utf-8")
if not text.startswith("---\n"):
raise ValueError(f"{path.name} has no frontmatter")
_, frontmatter, body = text.split("---", 2)
record = yaml.safe_load(frontmatter) or {}
if not isinstance(record, dict):
raise ValueError(f"{path.name} frontmatter is not a mapping")
if body.strip() and not record.get("summary"):
record["summary"] = body.strip()
records.append(record)
return records, _digest(b"".join(parts))
def _entity_id(dataspace: str, kind: str, key: str) -> str:
return f"urn:ce:{dataspace}:{kind}:{key}"
def _unpublished(report: dict) -> dict[str, Any]:
return {"snapshot": None, "manifest": None, "report": report}
def build_manifest(
*,
dataspace_id: str,
heading: str,
kinds: list[dict],
relation_labels: dict[str, dict[str, str]],
predicates: list[str],
fact_policy: str = "show-all",
acyclic_predicates: list[str] | None = None,
) -> tuple[dict[str, Any] | None, list[str]]:
"""Build a v0.1 manifest. Missing relation labels are returned and not invented."""
missing = sorted({predicate for predicate in predicates if predicate not in relation_labels})
if missing:
return None, missing
kind_ids = [kind["id"] for kind in kinds]
labels = {
predicate: {
"forward": relation_labels[predicate]["forward"],
"inverse": relation_labels[predicate]["inverse"],
}
for predicate in sorted(set(predicates))
}
manifest = {
"schemaVersion": "0.1",
"id": dataspace_id,
"factPolicy": fact_policy,
"home": {"heading": heading, "featuredKinds": kind_ids},
"navigation": [{"label": "Browse", "kinds": kind_ids}],
"kindViews": {
kind["id"]: {"columns": ["title"], "facets": ["tags"]}
for kind in kinds
},
"panels": ["summary", "relations", "evidence", "claims"],
"relationLabels": labels,
"acyclicPredicates": list(acyclic_predicates or []),
}
return manifest, []
def human_import_report(report: dict) -> str:
"""Provenance and rights summary a person can read next to the snapshot."""
source = report.get("source") or {}
lines = [
"# Import report",
"",
f"Connector version: {report.get('connectorVersion', '')}",
f"Source id: {source.get('id', '')}",
f"Source URI: {source.get('uri', '')}",
f"License: {source.get('license', '')}",
f"Attribution: {source.get('attribution') or '(none)'}",
f"Source digest: {report.get('sourceDigest') or '(none)'}",
f"Content hash: {report.get('contentHash') or '(not published)'}",
f"Published: {str(bool(report.get('published'))).lower()}",
f"Candidates: {report.get('candidates', 0)}",
f"Accepted: {report.get('accepted', 0)}",
"",
"## Tombstones",
]
tombstones = report.get("tombstoned") or []
lines.extend(f"- {key}" for key in tombstones)
if not tombstones:
lines.append("- (none)")
lines.extend(["", "## Quarantine"])
quarantined = report.get("quarantined") or []
if not quarantined:
lines.append("- (none)")
for item in quarantined:
lines.append(f"- {json.dumps(item, sort_keys=True)}")
lines.extend(["", "## Dangling references"])
dangling = report.get("dangling") or []
if not dangling:
lines.append("- (none)")
for item in dangling:
lines.append(f"- {item.get('from')} -{item.get('predicate')}-> {item.get('to')}")
lines.extend(["", "## Identity conflicts"])
conflicts = report.get("conflicts") or []
if not conflicts:
lines.append("- (none)")
for item in conflicts:
lines.append(f"- {item.get('key')}: {item.get('reason')}")
lines.append("")
return "\n".join(lines)
def project_records(
records: list[dict],
*,
dataspace_id: str,
title: str,
description: str,
source: dict,
kinds: list[dict],
observed_at: str,
source_digest: str = "",
relation_labels: dict[str, dict[str, str]] | None = None,
fact_policy: str = "show-all",
acyclic_predicates: list[str] | None = None,
) -> dict[str, Any]:
"""Return a report. `snapshot` is present only when publication is allowed.
Live keys are collected first. Relations resolve in a second pass against
that set. Duplicate keys and a live key that is also tombstoned are conflicts.
"""
license_name = str(source.get("license") or "")
report: dict[str, Any] = {
"connectorVersion": CONNECTOR_VERSION,
"candidates": len(records),
"accepted": 0,
"tombstoned": [],
"quarantined": [],
"dangling": [],
"conflicts": [],
"published": False,
"sourceDigest": source_digest,
"source": {
"id": source.get("id", ""),
"uri": source.get("uri", ""),
"title": source.get("title", ""),
"license": license_name,
"attribution": source.get("attribution") or "",
},
}
if license_name not in ALLOWED_LICENSES:
report["quarantined"].append({
"reason": "rights",
"license": license_name or "(missing)",
})
return _unpublished(report)
if license_name == "CC-BY-4.0" and not str(source.get("attribution") or "").strip():
report["quarantined"].append({
"reason": "rights",
"license": license_name,
"detail": "missing-attribution",
})
return _unpublished(report)
kind_ids = {kind["id"] for kind in kinds}
tombstoned_keys = set()
for record in records:
if record.get("deleted"):
report["tombstoned"].append(record.get("key"))
tombstoned_keys.add(f"{record.get('kind')}:{record.get('key')}")
grouped: dict[str, list[dict]] = {}
for record in records:
if record.get("deleted"):
continue
kind = record["kind"]
key = record["key"]
qualified = f"{kind}:{key}"
if kind not in kind_ids:
report["quarantined"].append({"reason": "unknown-kind", "key": key, "kind": kind})
continue
if qualified in tombstoned_keys:
report["conflicts"].append({"key": qualified, "reason": "tombstone-and-live"})
continue
grouped.setdefault(qualified, []).append(record)
accepted: dict[str, dict] = {}
for qualified, group in grouped.items():
if len(group) > 1:
report["conflicts"].append({
"key": qualified,
"reason": "duplicate-identity",
"count": len(group),
})
continue
accepted[qualified] = group[0]
entities = []
relations = []
claims = []
predicates: list[str] = []
# Second pass: links resolve only against keys accepted in the first pass.
for qualified, record in accepted.items():
kind, key = qualified.split(":", 1)
entity_id = _entity_id(dataspace_id, kind, key)
for link in record.get("relations") or []:
target = link.get("to")
predicate = link.get("predicate")
if target not in accepted:
report["dangling"].append({"from": qualified, "to": target, "predicate": predicate})
claim_id = f"claim:{kind}:{key}:tldr"
entities.append({
"id": entity_id,
"kindId": kind,
"title": record["title"],
"tldr": record["tldr"],
"summary": record.get("summary") or record["tldr"],
"tags": list(record.get("tags") or []),
"sourceRefs": [source["id"]],
"claimRefs": [claim_id],
"asOf": observed_at,
})
claims.append({
"id": claim_id,
"subjectId": entity_id,
"key": "tldr",
"value": record["tldr"],
"status": "asserted",
"evidenceRefs": [source["id"]],
"observedAt": observed_at,
})
for index, link in enumerate(record.get("relations") or [], start=1):
target = link["to"]
predicate = link["predicate"]
predicates.append(predicate)
target_kind, target_key = target.split(":", 1)
relations.append({
"id": f"{kind}-{key}-{index}",
"predicate": predicate,
"from": entity_id,
"to": _entity_id(dataspace_id, target_kind, target_key),
"sourceRefs": [source["id"]],
"asOf": observed_at,
})
blocked = (
report["dangling"]
or report["conflicts"]
or any(item["reason"] == "unknown-kind" for item in report["quarantined"])
)
if blocked:
return _unpublished(report)
manifest, missing_labels = build_manifest(
dataspace_id=dataspace_id,
heading=title,
kinds=kinds,
relation_labels=relation_labels or {},
predicates=predicates,
fact_policy=fact_policy,
acyclic_predicates=acyclic_predicates,
)
if missing_labels:
report["quarantined"].append({
"reason": "missing-relation-label",
"predicates": missing_labels,
})
return _unpublished(report)
entities.sort(key=lambda item: item["id"])
relations.sort(key=lambda item: item["id"])
claims.sort(key=lambda item: item["id"])
snapshot = {
"schemaVersion": "0.1",
"dataspace": {"id": dataspace_id, "title": title, "description": description},
"kinds": kinds,
"entities": entities,
"relations": relations,
"sources": [source],
"claims": claims,
}
for claim in claims:
if not claim["evidenceRefs"]:
report["quarantined"].append({"reason": "missing-evidence", "claim": claim["id"]})
return _unpublished(report)
report["accepted"] = len(entities)
report["published"] = True
report["contentHash"] = snapshot_hash(snapshot)
return {"snapshot": snapshot, "manifest": manifest, "report": report}
def publish_directory(destination: Path, result: dict[str, Any]) -> None:
"""Replace `destination` only after the snapshot, manifest, and both reports exist."""
if not result["report"]["published"] or result["snapshot"] is None or result.get("manifest") is None:
raise RuntimeError("refusing to publish a quarantined or dangling import")
destination.parent.mkdir(parents=True, exist_ok=True)
temporary = destination.parent / f".{destination.name}.incoming"
if temporary.exists():
shutil.rmtree(temporary)
temporary.mkdir()
(temporary / "snapshot.json").write_text(
json.dumps(result["snapshot"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "manifest.json").write_text(
json.dumps(result["manifest"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "import-report.json").write_text(
json.dumps(result["report"], indent=2) + "\n",
encoding="utf-8",
)
(temporary / "import-report.md").write_text(
human_import_report(result["report"]),
encoding="utf-8",
)
backup = destination.parent / f".{destination.name}.backup"
if backup.exists():
shutil.rmtree(backup)
if destination.exists():
os.rename(destination, backup)
os.rename(temporary, destination)

60
content_explorer/render.py Normal file
View file

@ -0,0 +1,60 @@
"""Render a snapshot with whatever labels the manifest supplies.
The text below is the shared explorer. It must not name a dataspace, a kind,
or a predicate. Those strings come from the snapshot and the manifest.
"""
from __future__ import annotations
from content_explorer.contract import resolve_claims
def render_explorer(snapshot: dict, manifest: dict) -> str:
kinds = {kind["id"]: kind for kind in snapshot.get("kinds", [])}
entities = {entity["id"]: entity for entity in snapshot.get("entities", [])}
claims = snapshot.get("claims", [])
policy = manifest.get("factPolicy", "show-all")
lines: list[str] = [manifest["home"]["heading"], ""]
for section in manifest.get("navigation", []):
names = [kinds[kind_id]["plural"] for kind_id in section["kinds"] if kind_id in kinds]
lines.append(f"{section['label']}: {', '.join(names)}")
lines.append("")
for kind_id in manifest["home"]["featuredKinds"]:
kind = kinds.get(kind_id)
if kind is None:
continue
lines.append(f"## {kind['plural']}")
members = [entity for entity in snapshot.get("entities", []) if entity.get("kindId") == kind_id]
if not members:
lines.append("(none)")
for entity in members:
field = resolve_claims(
[claim for claim in claims if claim.get("subjectId") == entity["id"]],
"tldr",
policy,
)
if field["state"] == "unknown":
shown = "unknown"
elif field["state"] == "contradicted":
shown = "contested: " + " | ".join(str(value) for value in field["values"])
elif field["state"] == "withheld":
shown = "withheld"
elif field["values"]:
shown = str(field["values"][0])
else:
shown = entity.get("tldr", "")
lines.append(f"- {entity['title']}: {shown}")
lines.append("")
lines.append("## Relations")
labels = manifest.get("relationLabels") or {}
for relation in snapshot.get("relations", []):
predicate = relation["predicate"]
wording = labels.get(predicate, {}).get("forward", predicate)
origin = entities.get(relation["from"], {}).get("title", relation["from"])
destination = entities.get(relation["to"], {}).get("title", relation["to"])
lines.append(f"- {origin} — {wording} → {destination}")
lines.append("")
return "\n".join(lines).rstrip() + "\n"