"""Build a v0.1 snapshot from JSON, NDJSON, or Markdown frontmatter. Connectors stop at a snapshot and a report. They do not render. """ from __future__ import annotations import hashlib import json import os import shutil from pathlib import Path from typing import Any import yaml CONNECTOR_VERSION = "0.1.0" ALLOWED_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "MIT"}) def _digest(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() def snapshot_hash(snapshot: dict) -> str: encoded = json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode("utf-8") return _digest(encoded) def load_json_records(path: Path) -> tuple[list[dict], str]: raw = path.read_bytes() document = json.loads(raw.decode("utf-8")) if isinstance(document, dict) and "records" in document: records = document["records"] elif isinstance(document, list): records = document else: raise ValueError(f"{path} is not a record list") return records, _digest(raw) def load_ndjson_records(path: Path) -> tuple[list[dict], str]: raw = path.read_bytes() records = [] for line in raw.decode("utf-8").splitlines(): if line.strip(): records.append(json.loads(line)) return records, _digest(raw) def load_markdown_records(directory: Path) -> tuple[list[dict], str]: parts: list[bytes] = [] records: list[dict] = [] for path in sorted(directory.glob("*.md")): raw = path.read_bytes() parts.append(raw) text = raw.decode("utf-8") if not text.startswith("---\n"): raise ValueError(f"{path.name} has no frontmatter") _, frontmatter, body = text.split("---", 2) record = yaml.safe_load(frontmatter) or {} if not isinstance(record, dict): raise ValueError(f"{path.name} frontmatter is not a mapping") if body.strip() and not record.get("summary"): record["summary"] = body.strip() records.append(record) return records, _digest(b"".join(parts)) def _entity_id(dataspace: str, kind: str, key: str) -> str: return f"urn:ce:{dataspace}:{kind}:{key}" def _unpublished(report: dict) -> dict[str, Any]: return {"snapshot": None, "manifest": None, "report": report} def build_manifest( *, dataspace_id: str, heading: str, kinds: list[dict], relation_labels: dict[str, dict[str, str]], predicates: list[str], fact_policy: str = "show-all", acyclic_predicates: list[str] | None = None, ) -> tuple[dict[str, Any] | None, list[str]]: """Build a v0.1 manifest. Missing relation labels are returned and not invented.""" missing = sorted({predicate for predicate in predicates if predicate not in relation_labels}) if missing: return None, missing kind_ids = [kind["id"] for kind in kinds] labels = { predicate: { "forward": relation_labels[predicate]["forward"], "inverse": relation_labels[predicate]["inverse"], } for predicate in sorted(set(predicates)) } manifest = { "schemaVersion": "0.1", "id": dataspace_id, "factPolicy": fact_policy, "home": {"heading": heading, "featuredKinds": kind_ids}, "navigation": [{"label": "Browse", "kinds": kind_ids}], "kindViews": { kind["id"]: {"columns": ["title"], "facets": ["tags"]} for kind in kinds }, "panels": ["summary", "relations", "evidence", "claims"], "relationLabels": labels, "acyclicPredicates": list(acyclic_predicates or []), } return manifest, [] def human_import_report(report: dict) -> str: """Provenance and rights summary a person can read next to the snapshot.""" source = report.get("source") or {} lines = [ "# Import report", "", f"Connector version: {report.get('connectorVersion', '')}", f"Source id: {source.get('id', '')}", f"Source URI: {source.get('uri', '')}", f"License: {source.get('license', '')}", f"Attribution: {source.get('attribution') or '(none)'}", f"Source digest: {report.get('sourceDigest') or '(none)'}", f"Content hash: {report.get('contentHash') or '(not published)'}", f"Published: {str(bool(report.get('published'))).lower()}", f"Candidates: {report.get('candidates', 0)}", f"Accepted: {report.get('accepted', 0)}", "", "## Tombstones", ] tombstones = report.get("tombstoned") or [] lines.extend(f"- {key}" for key in tombstones) if not tombstones: lines.append("- (none)") lines.extend(["", "## Quarantine"]) quarantined = report.get("quarantined") or [] if not quarantined: lines.append("- (none)") for item in quarantined: lines.append(f"- {json.dumps(item, sort_keys=True)}") lines.extend(["", "## Dangling references"]) dangling = report.get("dangling") or [] if not dangling: lines.append("- (none)") for item in dangling: lines.append(f"- {item.get('from')} -{item.get('predicate')}-> {item.get('to')}") lines.extend(["", "## Identity conflicts"]) conflicts = report.get("conflicts") or [] if not conflicts: lines.append("- (none)") for item in conflicts: lines.append(f"- {item.get('key')}: {item.get('reason')}") lines.append("") return "\n".join(lines) def project_records( records: list[dict], *, dataspace_id: str, title: str, description: str, source: dict, kinds: list[dict], observed_at: str, source_digest: str = "", relation_labels: dict[str, dict[str, str]] | None = None, fact_policy: str = "show-all", acyclic_predicates: list[str] | None = None, ) -> dict[str, Any]: """Return a report. `snapshot` is present only when publication is allowed. Live keys are collected first. Relations resolve in a second pass against that set. Duplicate keys and a live key that is also tombstoned are conflicts. """ license_name = str(source.get("license") or "") report: dict[str, Any] = { "connectorVersion": CONNECTOR_VERSION, "candidates": len(records), "accepted": 0, "tombstoned": [], "quarantined": [], "dangling": [], "conflicts": [], "published": False, "sourceDigest": source_digest, "source": { "id": source.get("id", ""), "uri": source.get("uri", ""), "title": source.get("title", ""), "license": license_name, "attribution": source.get("attribution") or "", }, } if license_name not in ALLOWED_LICENSES: report["quarantined"].append({ "reason": "rights", "license": license_name or "(missing)", }) return _unpublished(report) if license_name == "CC-BY-4.0" and not str(source.get("attribution") or "").strip(): report["quarantined"].append({ "reason": "rights", "license": license_name, "detail": "missing-attribution", }) return _unpublished(report) kind_ids = {kind["id"] for kind in kinds} tombstoned_keys = set() for record in records: if record.get("deleted"): report["tombstoned"].append(record.get("key")) tombstoned_keys.add(f"{record.get('kind')}:{record.get('key')}") grouped: dict[str, list[dict]] = {} for record in records: if record.get("deleted"): continue kind = record["kind"] key = record["key"] qualified = f"{kind}:{key}" if kind not in kind_ids: report["quarantined"].append({"reason": "unknown-kind", "key": key, "kind": kind}) continue if qualified in tombstoned_keys: report["conflicts"].append({"key": qualified, "reason": "tombstone-and-live"}) continue grouped.setdefault(qualified, []).append(record) accepted: dict[str, dict] = {} for qualified, group in grouped.items(): if len(group) > 1: report["conflicts"].append({ "key": qualified, "reason": "duplicate-identity", "count": len(group), }) continue accepted[qualified] = group[0] entities = [] relations = [] claims = [] predicates: list[str] = [] # Second pass: links resolve only against keys accepted in the first pass. for qualified, record in accepted.items(): kind, key = qualified.split(":", 1) entity_id = _entity_id(dataspace_id, kind, key) for link in record.get("relations") or []: target = link.get("to") predicate = link.get("predicate") if target not in accepted: report["dangling"].append({"from": qualified, "to": target, "predicate": predicate}) claim_id = f"claim:{kind}:{key}:tldr" entities.append({ "id": entity_id, "kindId": kind, "title": record["title"], "tldr": record["tldr"], "summary": record.get("summary") or record["tldr"], "tags": list(record.get("tags") or []), "sourceRefs": [source["id"]], "claimRefs": [claim_id], "asOf": observed_at, }) claims.append({ "id": claim_id, "subjectId": entity_id, "key": "tldr", "value": record["tldr"], "status": "asserted", "evidenceRefs": [source["id"]], "observedAt": observed_at, }) for index, link in enumerate(record.get("relations") or [], start=1): target = link["to"] predicate = link["predicate"] predicates.append(predicate) target_kind, target_key = target.split(":", 1) relations.append({ "id": f"{kind}-{key}-{index}", "predicate": predicate, "from": entity_id, "to": _entity_id(dataspace_id, target_kind, target_key), "sourceRefs": [source["id"]], "asOf": observed_at, }) blocked = ( report["dangling"] or report["conflicts"] or any(item["reason"] == "unknown-kind" for item in report["quarantined"]) ) if blocked: return _unpublished(report) manifest, missing_labels = build_manifest( dataspace_id=dataspace_id, heading=title, kinds=kinds, relation_labels=relation_labels or {}, predicates=predicates, fact_policy=fact_policy, acyclic_predicates=acyclic_predicates, ) if missing_labels: report["quarantined"].append({ "reason": "missing-relation-label", "predicates": missing_labels, }) return _unpublished(report) entities.sort(key=lambda item: item["id"]) relations.sort(key=lambda item: item["id"]) claims.sort(key=lambda item: item["id"]) snapshot = { "schemaVersion": "0.1", "dataspace": {"id": dataspace_id, "title": title, "description": description}, "kinds": kinds, "entities": entities, "relations": relations, "sources": [source], "claims": claims, } for claim in claims: if not claim["evidenceRefs"]: report["quarantined"].append({"reason": "missing-evidence", "claim": claim["id"]}) return _unpublished(report) report["accepted"] = len(entities) report["published"] = True report["contentHash"] = snapshot_hash(snapshot) return {"snapshot": snapshot, "manifest": manifest, "report": report} def publish_directory(destination: Path, result: dict[str, Any]) -> None: """Replace `destination` only after the snapshot, manifest, and both reports exist.""" if not result["report"]["published"] or result["snapshot"] is None or result.get("manifest") is None: raise RuntimeError("refusing to publish a quarantined or dangling import") destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.parent / f".{destination.name}.incoming" if temporary.exists(): shutil.rmtree(temporary) temporary.mkdir() (temporary / "snapshot.json").write_text( json.dumps(result["snapshot"], indent=2) + "\n", encoding="utf-8", ) (temporary / "manifest.json").write_text( json.dumps(result["manifest"], indent=2) + "\n", encoding="utf-8", ) (temporary / "import-report.json").write_text( json.dumps(result["report"], indent=2) + "\n", encoding="utf-8", ) (temporary / "import-report.md").write_text( human_import_report(result["report"]), encoding="utf-8", ) backup = destination.parent / f".{destination.name}.backup" if backup.exists(): shutil.rmtree(backup) if destination.exists(): os.rename(destination, backup) os.rename(temporary, destination)