tegwick
072a7fa550
The snapshot and manifest schemas, both example dataspaces, and the shared text renderer now pass the fixture validator and the negative contract tests. JSON, NDJSON, and Markdown frontmatter publish a snapshot only when rights, identity, and references allow it. CONTENT-WP-0001 and CONTENT-WP-0003 are marked finished against those artifacts. CONTENT-WP-0000-T04 stays open. SCOPE.md and the 19:05 scope-against-intent note record the stop. Assistant: grok Assistant-Session: 01a130f1-abfb-7d71-9cd6-a8b75ec7f95a
371 lines
13 KiB
Python
371 lines
13 KiB
Python
"""Build a v0.1 snapshot from JSON, NDJSON, or Markdown frontmatter.
|
|
|
|
Connectors stop at a snapshot and a report. They do not render.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
CONNECTOR_VERSION = "0.1.0"
|
|
ALLOWED_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "MIT"})
|
|
|
|
|
|
def _digest(payload: bytes) -> str:
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def snapshot_hash(snapshot: dict) -> str:
|
|
encoded = json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
return _digest(encoded)
|
|
|
|
|
|
def load_json_records(path: Path) -> tuple[list[dict], str]:
|
|
raw = path.read_bytes()
|
|
document = json.loads(raw.decode("utf-8"))
|
|
if isinstance(document, dict) and "records" in document:
|
|
records = document["records"]
|
|
elif isinstance(document, list):
|
|
records = document
|
|
else:
|
|
raise ValueError(f"{path} is not a record list")
|
|
return records, _digest(raw)
|
|
|
|
|
|
def load_ndjson_records(path: Path) -> tuple[list[dict], str]:
|
|
raw = path.read_bytes()
|
|
records = []
|
|
for line in raw.decode("utf-8").splitlines():
|
|
if line.strip():
|
|
records.append(json.loads(line))
|
|
return records, _digest(raw)
|
|
|
|
|
|
def load_markdown_records(directory: Path) -> tuple[list[dict], str]:
|
|
parts: list[bytes] = []
|
|
records: list[dict] = []
|
|
for path in sorted(directory.glob("*.md")):
|
|
raw = path.read_bytes()
|
|
parts.append(raw)
|
|
text = raw.decode("utf-8")
|
|
if not text.startswith("---\n"):
|
|
raise ValueError(f"{path.name} has no frontmatter")
|
|
_, frontmatter, body = text.split("---", 2)
|
|
record = yaml.safe_load(frontmatter) or {}
|
|
if not isinstance(record, dict):
|
|
raise ValueError(f"{path.name} frontmatter is not a mapping")
|
|
if body.strip() and not record.get("summary"):
|
|
record["summary"] = body.strip()
|
|
records.append(record)
|
|
return records, _digest(b"".join(parts))
|
|
|
|
|
|
def _entity_id(dataspace: str, kind: str, key: str) -> str:
|
|
return f"urn:ce:{dataspace}:{kind}:{key}"
|
|
|
|
|
|
def _unpublished(report: dict) -> dict[str, Any]:
|
|
return {"snapshot": None, "manifest": None, "report": report}
|
|
|
|
|
|
def build_manifest(
|
|
*,
|
|
dataspace_id: str,
|
|
heading: str,
|
|
kinds: list[dict],
|
|
relation_labels: dict[str, dict[str, str]],
|
|
predicates: list[str],
|
|
fact_policy: str = "show-all",
|
|
acyclic_predicates: list[str] | None = None,
|
|
) -> tuple[dict[str, Any] | None, list[str]]:
|
|
"""Build a v0.1 manifest. Missing relation labels are returned and not invented."""
|
|
missing = sorted({predicate for predicate in predicates if predicate not in relation_labels})
|
|
if missing:
|
|
return None, missing
|
|
kind_ids = [kind["id"] for kind in kinds]
|
|
labels = {
|
|
predicate: {
|
|
"forward": relation_labels[predicate]["forward"],
|
|
"inverse": relation_labels[predicate]["inverse"],
|
|
}
|
|
for predicate in sorted(set(predicates))
|
|
}
|
|
manifest = {
|
|
"schemaVersion": "0.1",
|
|
"id": dataspace_id,
|
|
"factPolicy": fact_policy,
|
|
"home": {"heading": heading, "featuredKinds": kind_ids},
|
|
"navigation": [{"label": "Browse", "kinds": kind_ids}],
|
|
"kindViews": {
|
|
kind["id"]: {"columns": ["title"], "facets": ["tags"]}
|
|
for kind in kinds
|
|
},
|
|
"panels": ["summary", "relations", "evidence", "claims"],
|
|
"relationLabels": labels,
|
|
"acyclicPredicates": list(acyclic_predicates or []),
|
|
}
|
|
return manifest, []
|
|
|
|
|
|
def human_import_report(report: dict) -> str:
|
|
"""Provenance and rights summary a person can read next to the snapshot."""
|
|
source = report.get("source") or {}
|
|
lines = [
|
|
"# Import report",
|
|
"",
|
|
f"Connector version: {report.get('connectorVersion', '')}",
|
|
f"Source id: {source.get('id', '')}",
|
|
f"Source URI: {source.get('uri', '')}",
|
|
f"License: {source.get('license', '')}",
|
|
f"Attribution: {source.get('attribution') or '(none)'}",
|
|
f"Source digest: {report.get('sourceDigest') or '(none)'}",
|
|
f"Content hash: {report.get('contentHash') or '(not published)'}",
|
|
f"Published: {str(bool(report.get('published'))).lower()}",
|
|
f"Candidates: {report.get('candidates', 0)}",
|
|
f"Accepted: {report.get('accepted', 0)}",
|
|
"",
|
|
"## Tombstones",
|
|
]
|
|
tombstones = report.get("tombstoned") or []
|
|
lines.extend(f"- {key}" for key in tombstones)
|
|
if not tombstones:
|
|
lines.append("- (none)")
|
|
lines.extend(["", "## Quarantine"])
|
|
quarantined = report.get("quarantined") or []
|
|
if not quarantined:
|
|
lines.append("- (none)")
|
|
for item in quarantined:
|
|
lines.append(f"- {json.dumps(item, sort_keys=True)}")
|
|
lines.extend(["", "## Dangling references"])
|
|
dangling = report.get("dangling") or []
|
|
if not dangling:
|
|
lines.append("- (none)")
|
|
for item in dangling:
|
|
lines.append(f"- {item.get('from')} -{item.get('predicate')}-> {item.get('to')}")
|
|
lines.extend(["", "## Identity conflicts"])
|
|
conflicts = report.get("conflicts") or []
|
|
if not conflicts:
|
|
lines.append("- (none)")
|
|
for item in conflicts:
|
|
lines.append(f"- {item.get('key')}: {item.get('reason')}")
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def project_records(
|
|
records: list[dict],
|
|
*,
|
|
dataspace_id: str,
|
|
title: str,
|
|
description: str,
|
|
source: dict,
|
|
kinds: list[dict],
|
|
observed_at: str,
|
|
source_digest: str = "",
|
|
relation_labels: dict[str, dict[str, str]] | None = None,
|
|
fact_policy: str = "show-all",
|
|
acyclic_predicates: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return a report. `snapshot` is present only when publication is allowed.
|
|
|
|
Live keys are collected first. Relations resolve in a second pass against
|
|
that set. Duplicate keys and a live key that is also tombstoned are conflicts.
|
|
"""
|
|
license_name = str(source.get("license") or "")
|
|
report: dict[str, Any] = {
|
|
"connectorVersion": CONNECTOR_VERSION,
|
|
"candidates": len(records),
|
|
"accepted": 0,
|
|
"tombstoned": [],
|
|
"quarantined": [],
|
|
"dangling": [],
|
|
"conflicts": [],
|
|
"published": False,
|
|
"sourceDigest": source_digest,
|
|
"source": {
|
|
"id": source.get("id", ""),
|
|
"uri": source.get("uri", ""),
|
|
"title": source.get("title", ""),
|
|
"license": license_name,
|
|
"attribution": source.get("attribution") or "",
|
|
},
|
|
}
|
|
if license_name not in ALLOWED_LICENSES:
|
|
report["quarantined"].append({
|
|
"reason": "rights",
|
|
"license": license_name or "(missing)",
|
|
})
|
|
return _unpublished(report)
|
|
if license_name == "CC-BY-4.0" and not str(source.get("attribution") or "").strip():
|
|
report["quarantined"].append({
|
|
"reason": "rights",
|
|
"license": license_name,
|
|
"detail": "missing-attribution",
|
|
})
|
|
return _unpublished(report)
|
|
|
|
kind_ids = {kind["id"] for kind in kinds}
|
|
tombstoned_keys = set()
|
|
for record in records:
|
|
if record.get("deleted"):
|
|
report["tombstoned"].append(record.get("key"))
|
|
tombstoned_keys.add(f"{record.get('kind')}:{record.get('key')}")
|
|
|
|
grouped: dict[str, list[dict]] = {}
|
|
for record in records:
|
|
if record.get("deleted"):
|
|
continue
|
|
kind = record["kind"]
|
|
key = record["key"]
|
|
qualified = f"{kind}:{key}"
|
|
if kind not in kind_ids:
|
|
report["quarantined"].append({"reason": "unknown-kind", "key": key, "kind": kind})
|
|
continue
|
|
if qualified in tombstoned_keys:
|
|
report["conflicts"].append({"key": qualified, "reason": "tombstone-and-live"})
|
|
continue
|
|
grouped.setdefault(qualified, []).append(record)
|
|
|
|
accepted: dict[str, dict] = {}
|
|
for qualified, group in grouped.items():
|
|
if len(group) > 1:
|
|
report["conflicts"].append({
|
|
"key": qualified,
|
|
"reason": "duplicate-identity",
|
|
"count": len(group),
|
|
})
|
|
continue
|
|
accepted[qualified] = group[0]
|
|
|
|
entities = []
|
|
relations = []
|
|
claims = []
|
|
predicates: list[str] = []
|
|
# Second pass: links resolve only against keys accepted in the first pass.
|
|
for qualified, record in accepted.items():
|
|
kind, key = qualified.split(":", 1)
|
|
entity_id = _entity_id(dataspace_id, kind, key)
|
|
for link in record.get("relations") or []:
|
|
target = link.get("to")
|
|
predicate = link.get("predicate")
|
|
if target not in accepted:
|
|
report["dangling"].append({"from": qualified, "to": target, "predicate": predicate})
|
|
claim_id = f"claim:{kind}:{key}:tldr"
|
|
entities.append({
|
|
"id": entity_id,
|
|
"kindId": kind,
|
|
"title": record["title"],
|
|
"tldr": record["tldr"],
|
|
"summary": record.get("summary") or record["tldr"],
|
|
"tags": list(record.get("tags") or []),
|
|
"sourceRefs": [source["id"]],
|
|
"claimRefs": [claim_id],
|
|
"asOf": observed_at,
|
|
})
|
|
claims.append({
|
|
"id": claim_id,
|
|
"subjectId": entity_id,
|
|
"key": "tldr",
|
|
"value": record["tldr"],
|
|
"status": "asserted",
|
|
"evidenceRefs": [source["id"]],
|
|
"observedAt": observed_at,
|
|
})
|
|
for index, link in enumerate(record.get("relations") or [], start=1):
|
|
target = link["to"]
|
|
predicate = link["predicate"]
|
|
predicates.append(predicate)
|
|
target_kind, target_key = target.split(":", 1)
|
|
relations.append({
|
|
"id": f"{kind}-{key}-{index}",
|
|
"predicate": predicate,
|
|
"from": entity_id,
|
|
"to": _entity_id(dataspace_id, target_kind, target_key),
|
|
"sourceRefs": [source["id"]],
|
|
"asOf": observed_at,
|
|
})
|
|
|
|
blocked = (
|
|
report["dangling"]
|
|
or report["conflicts"]
|
|
or any(item["reason"] == "unknown-kind" for item in report["quarantined"])
|
|
)
|
|
if blocked:
|
|
return _unpublished(report)
|
|
|
|
manifest, missing_labels = build_manifest(
|
|
dataspace_id=dataspace_id,
|
|
heading=title,
|
|
kinds=kinds,
|
|
relation_labels=relation_labels or {},
|
|
predicates=predicates,
|
|
fact_policy=fact_policy,
|
|
acyclic_predicates=acyclic_predicates,
|
|
)
|
|
if missing_labels:
|
|
report["quarantined"].append({
|
|
"reason": "missing-relation-label",
|
|
"predicates": missing_labels,
|
|
})
|
|
return _unpublished(report)
|
|
|
|
entities.sort(key=lambda item: item["id"])
|
|
relations.sort(key=lambda item: item["id"])
|
|
claims.sort(key=lambda item: item["id"])
|
|
snapshot = {
|
|
"schemaVersion": "0.1",
|
|
"dataspace": {"id": dataspace_id, "title": title, "description": description},
|
|
"kinds": kinds,
|
|
"entities": entities,
|
|
"relations": relations,
|
|
"sources": [source],
|
|
"claims": claims,
|
|
}
|
|
for claim in claims:
|
|
if not claim["evidenceRefs"]:
|
|
report["quarantined"].append({"reason": "missing-evidence", "claim": claim["id"]})
|
|
return _unpublished(report)
|
|
report["accepted"] = len(entities)
|
|
report["published"] = True
|
|
report["contentHash"] = snapshot_hash(snapshot)
|
|
return {"snapshot": snapshot, "manifest": manifest, "report": report}
|
|
|
|
|
|
def publish_directory(destination: Path, result: dict[str, Any]) -> None:
|
|
"""Replace `destination` only after the snapshot, manifest, and both reports exist."""
|
|
if not result["report"]["published"] or result["snapshot"] is None or result.get("manifest") is None:
|
|
raise RuntimeError("refusing to publish a quarantined or dangling import")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = destination.parent / f".{destination.name}.incoming"
|
|
if temporary.exists():
|
|
shutil.rmtree(temporary)
|
|
temporary.mkdir()
|
|
(temporary / "snapshot.json").write_text(
|
|
json.dumps(result["snapshot"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "manifest.json").write_text(
|
|
json.dumps(result["manifest"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "import-report.json").write_text(
|
|
json.dumps(result["report"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "import-report.md").write_text(
|
|
human_import_report(result["report"]),
|
|
encoding="utf-8",
|
|
)
|
|
backup = destination.parent / f".{destination.name}.backup"
|
|
if backup.exists():
|
|
shutil.rmtree(backup)
|
|
if destination.exists():
|
|
os.rename(destination, backup)
|
|
os.rename(temporary, destination)
|