tegwick
55f6180b8e
All checks were successful
synthetic / suite (push) Successful in 1m2s
The connector maps native Q-ids to aliases under urn:ce ids, publishes only through the existing rights and reference gate, and leaves the real-user pilot open. Assistant: grok Assistant-Session: 01a15378-f601-7fe2-a143-8e1500a1d59c
375 lines
13 KiB
Python
375 lines
13 KiB
Python
"""Build a v0.1 snapshot from JSON, NDJSON, or Markdown frontmatter.
|
|
|
|
Connectors stop at a snapshot and a report. They do not render.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
CONNECTOR_VERSION = "0.1.0"
|
|
ALLOWED_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "MIT"})
|
|
|
|
|
|
def _digest(payload: bytes) -> str:
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def snapshot_hash(snapshot: dict) -> str:
|
|
encoded = json.dumps(snapshot, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
return _digest(encoded)
|
|
|
|
|
|
def load_json_records(path: Path) -> tuple[list[dict], str]:
|
|
raw = path.read_bytes()
|
|
document = json.loads(raw.decode("utf-8"))
|
|
if isinstance(document, dict) and "records" in document:
|
|
records = document["records"]
|
|
elif isinstance(document, list):
|
|
records = document
|
|
else:
|
|
raise ValueError(f"{path} is not a record list")
|
|
return records, _digest(raw)
|
|
|
|
|
|
def load_ndjson_records(path: Path) -> tuple[list[dict], str]:
|
|
raw = path.read_bytes()
|
|
records = []
|
|
for line in raw.decode("utf-8").splitlines():
|
|
if line.strip():
|
|
records.append(json.loads(line))
|
|
return records, _digest(raw)
|
|
|
|
|
|
def load_markdown_records(directory: Path) -> tuple[list[dict], str]:
|
|
parts: list[bytes] = []
|
|
records: list[dict] = []
|
|
for path in sorted(directory.glob("*.md")):
|
|
raw = path.read_bytes()
|
|
parts.append(raw)
|
|
text = raw.decode("utf-8")
|
|
if not text.startswith("---\n"):
|
|
raise ValueError(f"{path.name} has no frontmatter")
|
|
_, frontmatter, body = text.split("---", 2)
|
|
record = yaml.safe_load(frontmatter) or {}
|
|
if not isinstance(record, dict):
|
|
raise ValueError(f"{path.name} frontmatter is not a mapping")
|
|
if body.strip() and not record.get("summary"):
|
|
record["summary"] = body.strip()
|
|
records.append(record)
|
|
return records, _digest(b"".join(parts))
|
|
|
|
|
|
def _entity_id(dataspace: str, kind: str, key: str) -> str:
|
|
return f"urn:ce:{dataspace}:{kind}:{key}"
|
|
|
|
|
|
def _unpublished(report: dict) -> dict[str, Any]:
|
|
return {"snapshot": None, "manifest": None, "report": report}
|
|
|
|
|
|
def build_manifest(
|
|
*,
|
|
dataspace_id: str,
|
|
heading: str,
|
|
kinds: list[dict],
|
|
relation_labels: dict[str, dict[str, str]],
|
|
predicates: list[str],
|
|
fact_policy: str = "show-all",
|
|
acyclic_predicates: list[str] | None = None,
|
|
) -> tuple[dict[str, Any] | None, list[str]]:
|
|
"""Build a v0.1 manifest. Missing relation labels are returned and not invented."""
|
|
missing = sorted({predicate for predicate in predicates if predicate not in relation_labels})
|
|
if missing:
|
|
return None, missing
|
|
kind_ids = [kind["id"] for kind in kinds]
|
|
labels = {
|
|
predicate: {
|
|
"forward": relation_labels[predicate]["forward"],
|
|
"inverse": relation_labels[predicate]["inverse"],
|
|
}
|
|
for predicate in sorted(set(predicates))
|
|
}
|
|
manifest = {
|
|
"schemaVersion": "0.1",
|
|
"id": dataspace_id,
|
|
"factPolicy": fact_policy,
|
|
"home": {"heading": heading, "featuredKinds": kind_ids},
|
|
"navigation": [{"label": "Browse", "kinds": kind_ids}],
|
|
"kindViews": {
|
|
kind["id"]: {"columns": ["title"], "facets": ["tags"]}
|
|
for kind in kinds
|
|
},
|
|
"panels": ["summary", "relations", "evidence", "claims"],
|
|
"relationLabels": labels,
|
|
"acyclicPredicates": list(acyclic_predicates or []),
|
|
}
|
|
return manifest, []
|
|
|
|
|
|
def human_import_report(report: dict) -> str:
|
|
"""Provenance and rights summary a person can read next to the snapshot."""
|
|
source = report.get("source") or {}
|
|
lines = [
|
|
"# Import report",
|
|
"",
|
|
f"Connector version: {report.get('connectorVersion', '')}",
|
|
f"Source id: {source.get('id', '')}",
|
|
f"Source URI: {source.get('uri', '')}",
|
|
f"License: {source.get('license', '')}",
|
|
f"Attribution: {source.get('attribution') or '(none)'}",
|
|
f"Source digest: {report.get('sourceDigest') or '(none)'}",
|
|
f"Content hash: {report.get('contentHash') or '(not published)'}",
|
|
f"Published: {str(bool(report.get('published'))).lower()}",
|
|
f"Candidates: {report.get('candidates', 0)}",
|
|
f"Accepted: {report.get('accepted', 0)}",
|
|
"",
|
|
"## Tombstones",
|
|
]
|
|
tombstones = report.get("tombstoned") or []
|
|
lines.extend(f"- {key}" for key in tombstones)
|
|
if not tombstones:
|
|
lines.append("- (none)")
|
|
lines.extend(["", "## Quarantine"])
|
|
quarantined = report.get("quarantined") or []
|
|
if not quarantined:
|
|
lines.append("- (none)")
|
|
for item in quarantined:
|
|
lines.append(f"- {json.dumps(item, sort_keys=True)}")
|
|
lines.extend(["", "## Dangling references"])
|
|
dangling = report.get("dangling") or []
|
|
if not dangling:
|
|
lines.append("- (none)")
|
|
for item in dangling:
|
|
lines.append(f"- {item.get('from')} -{item.get('predicate')}-> {item.get('to')}")
|
|
lines.extend(["", "## Identity conflicts"])
|
|
conflicts = report.get("conflicts") or []
|
|
if not conflicts:
|
|
lines.append("- (none)")
|
|
for item in conflicts:
|
|
lines.append(f"- {item.get('key')}: {item.get('reason')}")
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def project_records(
|
|
records: list[dict],
|
|
*,
|
|
dataspace_id: str,
|
|
title: str,
|
|
description: str,
|
|
source: dict,
|
|
kinds: list[dict],
|
|
observed_at: str,
|
|
source_digest: str = "",
|
|
relation_labels: dict[str, dict[str, str]] | None = None,
|
|
fact_policy: str = "show-all",
|
|
acyclic_predicates: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return a report. `snapshot` is present only when publication is allowed.
|
|
|
|
Live keys are collected first. Relations resolve in a second pass against
|
|
that set. Duplicate keys and a live key that is also tombstoned are conflicts.
|
|
"""
|
|
license_name = str(source.get("license") or "")
|
|
report: dict[str, Any] = {
|
|
"connectorVersion": CONNECTOR_VERSION,
|
|
"candidates": len(records),
|
|
"accepted": 0,
|
|
"tombstoned": [],
|
|
"quarantined": [],
|
|
"dangling": [],
|
|
"conflicts": [],
|
|
"published": False,
|
|
"sourceDigest": source_digest,
|
|
"source": {
|
|
"id": source.get("id", ""),
|
|
"uri": source.get("uri", ""),
|
|
"title": source.get("title", ""),
|
|
"license": license_name,
|
|
"attribution": source.get("attribution") or "",
|
|
},
|
|
}
|
|
if license_name not in ALLOWED_LICENSES:
|
|
report["quarantined"].append({
|
|
"reason": "rights",
|
|
"license": license_name or "(missing)",
|
|
})
|
|
return _unpublished(report)
|
|
if license_name == "CC-BY-4.0" and not str(source.get("attribution") or "").strip():
|
|
report["quarantined"].append({
|
|
"reason": "rights",
|
|
"license": license_name,
|
|
"detail": "missing-attribution",
|
|
})
|
|
return _unpublished(report)
|
|
|
|
kind_ids = {kind["id"] for kind in kinds}
|
|
tombstoned_keys = set()
|
|
for record in records:
|
|
if record.get("deleted"):
|
|
report["tombstoned"].append(record.get("key"))
|
|
tombstoned_keys.add(f"{record.get('kind')}:{record.get('key')}")
|
|
|
|
grouped: dict[str, list[dict]] = {}
|
|
for record in records:
|
|
if record.get("deleted"):
|
|
continue
|
|
kind = record["kind"]
|
|
key = record["key"]
|
|
qualified = f"{kind}:{key}"
|
|
if kind not in kind_ids:
|
|
report["quarantined"].append({"reason": "unknown-kind", "key": key, "kind": kind})
|
|
continue
|
|
if qualified in tombstoned_keys:
|
|
report["conflicts"].append({"key": qualified, "reason": "tombstone-and-live"})
|
|
continue
|
|
grouped.setdefault(qualified, []).append(record)
|
|
|
|
accepted: dict[str, dict] = {}
|
|
for qualified, group in grouped.items():
|
|
if len(group) > 1:
|
|
report["conflicts"].append({
|
|
"key": qualified,
|
|
"reason": "duplicate-identity",
|
|
"count": len(group),
|
|
})
|
|
continue
|
|
accepted[qualified] = group[0]
|
|
|
|
entities = []
|
|
relations = []
|
|
claims = []
|
|
predicates: list[str] = []
|
|
# Second pass: links resolve only against keys accepted in the first pass.
|
|
for qualified, record in accepted.items():
|
|
kind, key = qualified.split(":", 1)
|
|
entity_id = _entity_id(dataspace_id, kind, key)
|
|
for link in record.get("relations") or []:
|
|
target = link.get("to")
|
|
predicate = link.get("predicate")
|
|
if target not in accepted:
|
|
report["dangling"].append({"from": qualified, "to": target, "predicate": predicate})
|
|
claim_id = f"claim:{kind}:{key}:tldr"
|
|
entity = {
|
|
"id": entity_id,
|
|
"kindId": kind,
|
|
"title": record["title"],
|
|
"tldr": record["tldr"],
|
|
"summary": record.get("summary") or record["tldr"],
|
|
"tags": list(record.get("tags") or []),
|
|
"sourceRefs": [source["id"]],
|
|
"claimRefs": [claim_id],
|
|
"asOf": observed_at,
|
|
}
|
|
aliases = [str(alias) for alias in (record.get("aliases") or []) if str(alias).strip()]
|
|
if aliases:
|
|
entity["aliases"] = aliases
|
|
entities.append(entity)
|
|
claims.append({
|
|
"id": claim_id,
|
|
"subjectId": entity_id,
|
|
"key": "tldr",
|
|
"value": record["tldr"],
|
|
"status": "asserted",
|
|
"evidenceRefs": [source["id"]],
|
|
"observedAt": observed_at,
|
|
})
|
|
for index, link in enumerate(record.get("relations") or [], start=1):
|
|
target = link["to"]
|
|
predicate = link["predicate"]
|
|
predicates.append(predicate)
|
|
target_kind, target_key = target.split(":", 1)
|
|
relations.append({
|
|
"id": f"{kind}-{key}-{index}",
|
|
"predicate": predicate,
|
|
"from": entity_id,
|
|
"to": _entity_id(dataspace_id, target_kind, target_key),
|
|
"sourceRefs": [source["id"]],
|
|
"asOf": observed_at,
|
|
})
|
|
|
|
blocked = (
|
|
report["dangling"]
|
|
or report["conflicts"]
|
|
or any(item["reason"] == "unknown-kind" for item in report["quarantined"])
|
|
)
|
|
if blocked:
|
|
return _unpublished(report)
|
|
|
|
manifest, missing_labels = build_manifest(
|
|
dataspace_id=dataspace_id,
|
|
heading=title,
|
|
kinds=kinds,
|
|
relation_labels=relation_labels or {},
|
|
predicates=predicates,
|
|
fact_policy=fact_policy,
|
|
acyclic_predicates=acyclic_predicates,
|
|
)
|
|
if missing_labels:
|
|
report["quarantined"].append({
|
|
"reason": "missing-relation-label",
|
|
"predicates": missing_labels,
|
|
})
|
|
return _unpublished(report)
|
|
|
|
entities.sort(key=lambda item: item["id"])
|
|
relations.sort(key=lambda item: item["id"])
|
|
claims.sort(key=lambda item: item["id"])
|
|
snapshot = {
|
|
"schemaVersion": "0.1",
|
|
"dataspace": {"id": dataspace_id, "title": title, "description": description},
|
|
"kinds": kinds,
|
|
"entities": entities,
|
|
"relations": relations,
|
|
"sources": [source],
|
|
"claims": claims,
|
|
}
|
|
for claim in claims:
|
|
if not claim["evidenceRefs"]:
|
|
report["quarantined"].append({"reason": "missing-evidence", "claim": claim["id"]})
|
|
return _unpublished(report)
|
|
report["accepted"] = len(entities)
|
|
report["published"] = True
|
|
report["contentHash"] = snapshot_hash(snapshot)
|
|
return {"snapshot": snapshot, "manifest": manifest, "report": report}
|
|
|
|
|
|
def publish_directory(destination: Path, result: dict[str, Any]) -> None:
|
|
"""Replace `destination` only after the snapshot, manifest, and both reports exist."""
|
|
if not result["report"]["published"] or result["snapshot"] is None or result.get("manifest") is None:
|
|
raise RuntimeError("refusing to publish a quarantined or dangling import")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = destination.parent / f".{destination.name}.incoming"
|
|
if temporary.exists():
|
|
shutil.rmtree(temporary)
|
|
temporary.mkdir()
|
|
(temporary / "snapshot.json").write_text(
|
|
json.dumps(result["snapshot"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "manifest.json").write_text(
|
|
json.dumps(result["manifest"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "import-report.json").write_text(
|
|
json.dumps(result["report"], indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
(temporary / "import-report.md").write_text(
|
|
human_import_report(result["report"]),
|
|
encoding="utf-8",
|
|
)
|
|
backup = destination.parent / f".{destination.name}.backup"
|
|
if backup.exists():
|
|
shutil.rmtree(backup)
|
|
if destination.exists():
|
|
os.rename(destination, backup)
|
|
os.rename(temporary, destination)
|