From 55f6180b8ec80bba8432423becfe304050b8405a Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 19 Oct 2026 11:38:04 +0200 Subject: [PATCH] Add a Wikidata export connector for a pinned CC0 computing field. The connector maps native Q-ids to aliases under urn:ce ids, publishes only through the existing rights and reference gate, and leaves the real-user pilot open. Assistant: grok Assistant-Session: 01a15378-f601-7fe2-a143-8e1500a1d59c --- connectors/AUTHORING.md | 12 + connectors/__init__.py | 1 + connectors/wikidata_export.py | 181 +++++ content_explorer/ingest.py | 8 +- .../computing-basics/import-report.json | 43 ++ .../computing-basics/import-report.md | 24 + .../dataspaces/computing-basics/manifest.json | 70 ++ .../dataspaces/computing-basics/snapshot.json | 636 ++++++++++++++++++ .../sources/wikidata/computing-basics.json | 625 +++++++++++++++++ tests/test_quality.py | 2 +- tests/test_wikidata_connector.py | 150 +++++ ui/components/both-dataspaces.test.tsx | 3 +- ui/lib/computing-basics.test.ts | 37 + workplans/CONTENT-WP-0006-external-pilot.md | 14 +- .../evidence/CONTENT-WP-0006-boundaries.md | 23 + 15 files changed, 1819 insertions(+), 10 deletions(-) create mode 100644 connectors/AUTHORING.md create mode 100644 connectors/__init__.py create mode 100644 connectors/wikidata_export.py create mode 100644 examples/dataspaces/computing-basics/import-report.json create mode 100644 examples/dataspaces/computing-basics/import-report.md create mode 100644 examples/dataspaces/computing-basics/manifest.json create mode 100644 examples/dataspaces/computing-basics/snapshot.json create mode 100644 examples/sources/wikidata/computing-basics.json create mode 100644 tests/test_wikidata_connector.py create mode 100644 ui/lib/computing-basics.test.ts create mode 100644 workplans/evidence/CONTENT-WP-0006-boundaries.md diff --git a/connectors/AUTHORING.md b/connectors/AUTHORING.md new file mode 100644 index 0000000..f29a7c8 --- /dev/null +++ b/connectors/AUTHORING.md @@ -0,0 +1,12 @@ +# Connector authoring + +A connector maps one source format to candidate records, then calls `project_records` and `publish_directory` in `content_explorer/ingest.py`. It does not render and it does not import UI code. Native source identifiers are aliases on `urn:ce:` ids. Publication stays fail-closed: a forbidden license, an unresolved reference, or an identity conflict returns no snapshot. + +Certify a connector with a pinned local file, not a live API: + +1. An accepted run publishes a snapshot, a manifest, and both import reports. Every claim cites a source whose license is `CC0-1.0`, `CC-BY-4.0`, or `MIT`. +2. A refused run (forbidden rights, unresolved reference, or identity conflict) publishes no snapshot. +3. Replaying the same file yields the same entity set. +4. A changed file refreshes the snapshot. The report records the connector version, the input digest, and the counts. + +`connectors/wikidata_export.py` and `tests/test_wikidata_connector.py` are the worked example. The input is `examples/sources/wikidata/computing-basics.json`. diff --git a/connectors/__init__.py b/connectors/__init__.py new file mode 100644 index 0000000..d8fd0d2 --- /dev/null +++ b/connectors/__init__.py @@ -0,0 +1 @@ +"""Source connectors. Each one maps an external format and does not render.""" diff --git a/connectors/wikidata_export.py b/connectors/wikidata_export.py new file mode 100644 index 0000000..36ad65a --- /dev/null +++ b/connectors/wikidata_export.py @@ -0,0 +1,181 @@ +"""Map a pinned Wikidata entity export into a canonical snapshot. + +This is the third connector. It does not use the JSON, NDJSON, or Markdown +loaders, and it does not import UI code. Candidates go through +``project_records`` and ``publish_directory``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from content_explorer.ingest import project_records, publish_directory, snapshot_hash + +CONNECTOR_ID = "wikidata-export" +CONNECTOR_VERSION = "0.1.0" +SOURCE_NAMESPACE = "https://www.wikidata.org/wiki/" +MAPPING_VERSION = "1" +PROPERTY_PREDICATES = {"P287": "designed-by"} +FORMAT = "wikidata-entity-export" + + +def _digest(payload: bytes) -> str: + import hashlib + + return hashlib.sha256(payload).hexdigest() + + +def _en(block: dict | None) -> str: + entry = (block or {}).get("en") or {} + if isinstance(entry, dict): + return str(entry.get("value") or "").strip() + return "" + + +def _target_qid(statement: dict) -> str | None: + snak = statement.get("mainsnak") or {} + if snak.get("snaktype") != "value": + return None + value = (snak.get("datavalue") or {}).get("value") + if not isinstance(value, dict): + return None + qid = value.get("id") + if isinstance(qid, str) and len(qid) > 1 and qid[0] == "Q" and qid[1:].isdigit(): + return qid + return None + + +def load_export(path: Path) -> tuple[dict[str, Any], str]: + """Read one pinned Wikidata export. The digest is of the file bytes.""" + raw = path.read_bytes() + document = json.loads(raw.decode("utf-8")) + if not isinstance(document, dict) or document.get("format") != FORMAT: + raise ValueError(f"{path} is not a {FORMAT}") + if not isinstance(document.get("entities"), dict): + raise ValueError(f"{path} has no Wikidata entity map") + return document, _digest(raw) + + +def map_candidates(document: dict[str, Any]) -> list[dict[str, Any]]: + """Turn Wikidata items into candidate records. Native Q-ids become aliases.""" + entities = document["entities"] + bindings = document.get("kindByQid") or {} + if not isinstance(bindings, dict) or not bindings: + raise ValueError("kindByQid is required") + located = { + qid: f"{binding['kind']}:{binding['key']}" + for qid, binding in bindings.items() + } + records: list[dict[str, Any]] = [] + for qid, binding in bindings.items(): + item = entities.get(qid) + if not isinstance(item, dict): + raise ValueError(f"{qid} is not in the export entities") + title = _en(item.get("labels")) + description = _en(item.get("descriptions")) + if not title or not description: + raise ValueError(f"{qid} needs an English label and description") + aliases = [qid, f"{SOURCE_NAMESPACE}{qid}"] + for alias in (item.get("aliases") or {}).get("en") or []: + value = alias.get("value") if isinstance(alias, dict) else str(alias) + value = str(value).strip() + if value and value not in aliases: + aliases.append(value) + relations = [] + claims = item.get("claims") or {} + for prop, predicate in PROPERTY_PREDICATES.items(): + for statement in claims.get(prop) or []: + target = _target_qid(statement) + if not target: + continue + relations.append({ + "predicate": predicate, + "to": located.get(target) or f"missing:{target.lower()}", + }) + records.append({ + "kind": binding["kind"], + "key": binding["key"], + "title": title, + "tldr": description, + "summary": description, + "tags": [binding["kind"]], + "aliases": aliases, + "relations": relations, + }) + return records + + +def _source(document: dict[str, Any]) -> dict[str, str]: + raw = document.get("source") or {} + source = { + "id": str(raw.get("id") or ""), + "uri": str(raw.get("uri") or ""), + "title": str(raw.get("title") or ""), + "license": str(raw.get("license") or ""), + "attribution": str(raw.get("attribution") or ""), + } + retrieved = str(document.get("retrievedAt") or "") + if retrieved: + source["retrievedAt"] = retrieved + return source + + +def _annotate(report: dict[str, Any], digest: str) -> None: + report["connectorVersion"] = f"{CONNECTOR_ID}/{CONNECTOR_VERSION}" + report["sourceDigest"] = digest + report["inputDigest"] = digest + report["connector"] = { + "id": CONNECTOR_ID, + "version": CONNECTOR_VERSION, + "sourceNamespace": SOURCE_NAMESPACE, + "capabilities": ["map", "validate", "commit"], + "licensePolicy": "CC0-1.0, CC-BY-4.0, MIT", + "mappingVersion": MAPPING_VERSION, + "refreshMode": "pinned-export-replay", + "sourcePriority": 1, + "lastCheckpoint": digest, + } + report["counts"] = { + "candidates": report.get("candidates", 0), + "accepted": report.get("accepted", 0), + "tombstoned": len(report.get("tombstoned") or []), + "quarantined": len(report.get("quarantined") or []), + "dangling": len(report.get("dangling") or []), + "conflicts": len(report.get("conflicts") or []), + } + + +def _browsable_columns(manifest: dict[str, Any] | None) -> None: + if not manifest: + return + for view in (manifest.get("kindViews") or {}).values(): + view["columns"] = ["name", "tldr", "tags"] + view["facets"] = ["tags"] + + +def run(export_path: Path, destination: Path | None = None) -> dict[str, Any]: + """Map ``export_path`` and publish only when the existing gate allows it.""" + document, digest = load_export(export_path) + dataspace = document["dataspace"] + result = project_records( + map_candidates(document), + dataspace_id=dataspace["id"], + title=dataspace["title"], + description=dataspace["description"], + source=_source(document), + kinds=list(document["kinds"]), + observed_at=str(document["retrievedAt"]), + source_digest=digest, + relation_labels=document.get("relationLabels") or {}, + fact_policy=document.get("factPolicy") or "show-all", + acyclic_predicates=list(document.get("acyclicPredicates") or []), + ) + _annotate(result["report"], digest) + _browsable_columns(result.get("manifest")) + if result["report"]["published"] and result.get("snapshot") is not None: + result["report"]["contentHash"] = snapshot_hash(result["snapshot"]) + if destination is not None and result["report"]["published"]: + publish_directory(destination, result) + return result diff --git a/content_explorer/ingest.py b/content_explorer/ingest.py index fb15e11..5bea172 100644 --- a/content_explorer/ingest.py +++ b/content_explorer/ingest.py @@ -258,7 +258,7 @@ def project_records( if target not in accepted: report["dangling"].append({"from": qualified, "to": target, "predicate": predicate}) claim_id = f"claim:{kind}:{key}:tldr" - entities.append({ + entity = { "id": entity_id, "kindId": kind, "title": record["title"], @@ -268,7 +268,11 @@ def project_records( "sourceRefs": [source["id"]], "claimRefs": [claim_id], "asOf": observed_at, - }) + } + aliases = [str(alias) for alias in (record.get("aliases") or []) if str(alias).strip()] + if aliases: + entity["aliases"] = aliases + entities.append(entity) claims.append({ "id": claim_id, "subjectId": entity_id, diff --git a/examples/dataspaces/computing-basics/import-report.json b/examples/dataspaces/computing-basics/import-report.json new file mode 100644 index 0000000..2447069 --- /dev/null +++ b/examples/dataspaces/computing-basics/import-report.json @@ -0,0 +1,43 @@ +{ + "connectorVersion": "wikidata-export/0.1.0", + "candidates": 16, + "accepted": 16, + "tombstoned": [], + "quarantined": [], + "dangling": [], + "conflicts": [], + "published": true, + "sourceDigest": "e506c3563820b18c84f1d7670dd455b4f53bd0431849fa97ee78258e16b6a821", + "source": { + "id": "src:wikidata:computing-basics", + "uri": "https://www.wikidata.org/wiki/Wikidata:Licensing", + "title": "Wikidata", + "license": "CC0-1.0", + "attribution": "Wikidata contributors" + }, + "contentHash": "76fc628d0f56dd5ea9bf164b83ad70f7d8b1f42943c71ced2948808acee91d75", + "inputDigest": "e506c3563820b18c84f1d7670dd455b4f53bd0431849fa97ee78258e16b6a821", + "connector": { + "id": "wikidata-export", + "version": "0.1.0", + "sourceNamespace": "https://www.wikidata.org/wiki/", + "capabilities": [ + "map", + "validate", + "commit" + ], + "licensePolicy": "CC0-1.0, CC-BY-4.0, MIT", + "mappingVersion": "1", + "refreshMode": "pinned-export-replay", + "sourcePriority": 1, + "lastCheckpoint": "e506c3563820b18c84f1d7670dd455b4f53bd0431849fa97ee78258e16b6a821" + }, + "counts": { + "candidates": 16, + "accepted": 16, + "tombstoned": 0, + "quarantined": 0, + "dangling": 0, + "conflicts": 0 + } +} diff --git a/examples/dataspaces/computing-basics/import-report.md b/examples/dataspaces/computing-basics/import-report.md new file mode 100644 index 0000000..44c09f8 --- /dev/null +++ b/examples/dataspaces/computing-basics/import-report.md @@ -0,0 +1,24 @@ +# Import report + +Connector version: wikidata-export/0.1.0 +Source id: src:wikidata:computing-basics +Source URI: https://www.wikidata.org/wiki/Wikidata:Licensing +License: CC0-1.0 +Attribution: Wikidata contributors +Source digest: e506c3563820b18c84f1d7670dd455b4f53bd0431849fa97ee78258e16b6a821 +Content hash: 76fc628d0f56dd5ea9bf164b83ad70f7d8b1f42943c71ced2948808acee91d75 +Published: true +Candidates: 16 +Accepted: 16 + +## Tombstones +- (none) + +## Quarantine +- (none) + +## Dangling references +- (none) + +## Identity conflicts +- (none) diff --git a/examples/dataspaces/computing-basics/manifest.json b/examples/dataspaces/computing-basics/manifest.json new file mode 100644 index 0000000..6d01594 --- /dev/null +++ b/examples/dataspaces/computing-basics/manifest.json @@ -0,0 +1,70 @@ +{ + "schemaVersion": "0.1", + "id": "computing-basics", + "factPolicy": "show-all", + "home": { + "heading": "Computing basics", + "featuredKinds": [ + "programming-language", + "language-designer", + "formalism" + ] + }, + "navigation": [ + { + "label": "Browse", + "kinds": [ + "programming-language", + "language-designer", + "formalism" + ] + } + ], + "kindViews": { + "programming-language": { + "columns": [ + "name", + "tldr", + "tags" + ], + "facets": [ + "tags" + ] + }, + "language-designer": { + "columns": [ + "name", + "tldr", + "tags" + ], + "facets": [ + "tags" + ] + }, + "formalism": { + "columns": [ + "name", + "tldr", + "tags" + ], + "facets": [ + "tags" + ] + } + }, + "panels": [ + "summary", + "relations", + "evidence", + "claims" + ], + "relationLabels": { + "designed-by": { + "forward": "Designed by", + "inverse": "Designer of" + } + }, + "acyclicPredicates": [ + "designed-by" + ] +} diff --git a/examples/dataspaces/computing-basics/snapshot.json b/examples/dataspaces/computing-basics/snapshot.json new file mode 100644 index 0000000..e433ae9 --- /dev/null +++ b/examples/dataspaces/computing-basics/snapshot.json @@ -0,0 +1,636 @@ +{ + "schemaVersion": "0.1", + "dataspace": { + "id": "computing-basics", + "title": "Computing basics", + "description": "A pinned CC0 subset of Wikidata items about programming languages, the people who designed them, and two formalisms." + }, + "kinds": [ + { + "id": "programming-language", + "label": "Programming language", + "plural": "Programming languages" + }, + { + "id": "language-designer", + "label": "Language designer", + "plural": "Language designers" + }, + { + "id": "formalism", + "label": "Formalism", + "plural": "Formalisms" + } + ], + "entities": [ + { + "id": "urn:ce:computing-basics:formalism:algorithm", + "kindId": "formalism", + "title": "algorithm", + "tldr": "sequence of instructions to perform a task", + "summary": "sequence of instructions to perform a task", + "tags": [ + "formalism" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:formalism:algorithm:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q8366", + "https://www.wikidata.org/wiki/Q8366", + "algorithms" + ] + }, + { + "id": "urn:ce:computing-basics:formalism:programming-language", + "kindId": "formalism", + "title": "programming language", + "tldr": "language for communicating instructions to a machine", + "summary": "language for communicating instructions to a machine", + "tags": [ + "formalism" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:formalism:programming-language:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q9143", + "https://www.wikidata.org/wiki/Q9143", + "computer language" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:bjarne-stroustrup", + "kindId": "language-designer", + "title": "Bjarne Stroustrup", + "tldr": "Danish computer scientist, creator of C++ (born 1950)", + "summary": "Danish computer scientist, creator of C++ (born 1950)", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:bjarne-stroustrup:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q92620", + "https://www.wikidata.org/wiki/Q92620" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:brendan-eich", + "kindId": "language-designer", + "title": "Brendan Eich", + "tldr": "American computer programmer and technology executive", + "summary": "American computer programmer and technology executive", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:brendan-eich:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q92648", + "https://www.wikidata.org/wiki/Q92648", + "Brendan O. Eich" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:dennis-ritchie", + "kindId": "language-designer", + "title": "Dennis M. Ritchie", + "tldr": "American computer scientist, co-inventor of the Unix operating system and the C programming language (1941\u20132011)", + "summary": "American computer scientist, co-inventor of the Unix operating system and the C programming language (1941\u20132011)", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:dennis-ritchie:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q45575", + "https://www.wikidata.org/wiki/Q45575", + "Dennis MacAlistair Ritchie" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:guido-van-rossum", + "kindId": "language-designer", + "title": "Guido van Rossum", + "tldr": "Dutch programmer and creator of Python", + "summary": "Dutch programmer and creator of Python", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:guido-van-rossum:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q30942", + "https://www.wikidata.org/wiki/Q30942", + "gvanrossum" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:james-gosling", + "kindId": "language-designer", + "title": "James Gosling", + "tldr": "Canadian computer scientist", + "summary": "Canadian computer scientist", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:james-gosling:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q92622", + "https://www.wikidata.org/wiki/Q92622", + "James Arthur Gosling" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:rasmus-lerdorf", + "kindId": "language-designer", + "title": "Rasmus Lerdorf", + "tldr": "Danish programmer and creator of PHP", + "summary": "Danish programmer and creator of PHP", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:rasmus-lerdorf:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q92768", + "https://www.wikidata.org/wiki/Q92768", + "Rasmus Jay Lerdorf" + ] + }, + { + "id": "urn:ce:computing-basics:language-designer:yukihiro-matsumoto", + "kindId": "language-designer", + "title": "Yukihiro Matsumoto", + "tldr": "Japanese computer scientist who created the Ruby programming language", + "summary": "Japanese computer scientist who created the Ruby programming language", + "tags": [ + "language-designer" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:language-designer:yukihiro-matsumoto:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q92748", + "https://www.wikidata.org/wiki/Q92748", + "Matz" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:c", + "kindId": "programming-language", + "title": "C", + "tldr": "general-purpose programming language", + "summary": "general-purpose programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:c:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q15777", + "https://www.wikidata.org/wiki/Q15777", + "C programming language" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:cpp", + "kindId": "programming-language", + "title": "C++", + "tldr": "general-purpose programming language", + "summary": "general-purpose programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:cpp:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q2407", + "https://www.wikidata.org/wiki/Q2407" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:java", + "kindId": "programming-language", + "title": "Java", + "tldr": "object-oriented programming language", + "summary": "object-oriented programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:java:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q251", + "https://www.wikidata.org/wiki/Q251", + "java programming language" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:javascript", + "kindId": "programming-language", + "title": "JavaScript", + "tldr": "high-level programming language", + "summary": "high-level programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:javascript:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q2005", + "https://www.wikidata.org/wiki/Q2005" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:php", + "kindId": "programming-language", + "title": "PHP", + "tldr": "scripting language focused on web development", + "summary": "scripting language focused on web development", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:php:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q59", + "https://www.wikidata.org/wiki/Q59", + "PHP programming language" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:python", + "kindId": "programming-language", + "title": "Python", + "tldr": "general-purpose programming language", + "summary": "general-purpose programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:python:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q28865", + "https://www.wikidata.org/wiki/Q28865", + "Python programming language" + ] + }, + { + "id": "urn:ce:computing-basics:programming-language:ruby", + "kindId": "programming-language", + "title": "Ruby", + "tldr": "general-purpose programming language", + "summary": "general-purpose programming language", + "tags": [ + "programming-language" + ], + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "claimRefs": [ + "claim:programming-language:ruby:tldr" + ], + "asOf": "2026-10-19", + "aliases": [ + "Q161053", + "https://www.wikidata.org/wiki/Q161053", + "Ruby programming language" + ] + } + ], + "relations": [ + { + "id": "programming-language-c-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:c", + "to": "urn:ce:computing-basics:language-designer:dennis-ritchie", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-cpp-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:cpp", + "to": "urn:ce:computing-basics:language-designer:bjarne-stroustrup", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-java-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:java", + "to": "urn:ce:computing-basics:language-designer:james-gosling", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-javascript-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:javascript", + "to": "urn:ce:computing-basics:language-designer:brendan-eich", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-php-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:php", + "to": "urn:ce:computing-basics:language-designer:rasmus-lerdorf", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-python-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:python", + "to": "urn:ce:computing-basics:language-designer:guido-van-rossum", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + }, + { + "id": "programming-language-ruby-1", + "predicate": "designed-by", + "from": "urn:ce:computing-basics:programming-language:ruby", + "to": "urn:ce:computing-basics:language-designer:yukihiro-matsumoto", + "sourceRefs": [ + "src:wikidata:computing-basics" + ], + "asOf": "2026-10-19" + } + ], + "sources": [ + { + "id": "src:wikidata:computing-basics", + "uri": "https://www.wikidata.org/wiki/Wikidata:Licensing", + "title": "Wikidata", + "license": "CC0-1.0", + "attribution": "Wikidata contributors", + "retrievedAt": "2026-10-19" + } + ], + "claims": [ + { + "id": "claim:formalism:algorithm:tldr", + "subjectId": "urn:ce:computing-basics:formalism:algorithm", + "key": "tldr", + "value": "sequence of instructions to perform a task", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:formalism:programming-language:tldr", + "subjectId": "urn:ce:computing-basics:formalism:programming-language", + "key": "tldr", + "value": "language for communicating instructions to a machine", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:bjarne-stroustrup:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:bjarne-stroustrup", + "key": "tldr", + "value": "Danish computer scientist, creator of C++ (born 1950)", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:brendan-eich:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:brendan-eich", + "key": "tldr", + "value": "American computer programmer and technology executive", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:dennis-ritchie:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:dennis-ritchie", + "key": "tldr", + "value": "American computer scientist, co-inventor of the Unix operating system and the C programming language (1941\u20132011)", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:guido-van-rossum:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:guido-van-rossum", + "key": "tldr", + "value": "Dutch programmer and creator of Python", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:james-gosling:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:james-gosling", + "key": "tldr", + "value": "Canadian computer scientist", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:rasmus-lerdorf:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:rasmus-lerdorf", + "key": "tldr", + "value": "Danish programmer and creator of PHP", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:language-designer:yukihiro-matsumoto:tldr", + "subjectId": "urn:ce:computing-basics:language-designer:yukihiro-matsumoto", + "key": "tldr", + "value": "Japanese computer scientist who created the Ruby programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:c:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:c", + "key": "tldr", + "value": "general-purpose programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:cpp:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:cpp", + "key": "tldr", + "value": "general-purpose programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:java:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:java", + "key": "tldr", + "value": "object-oriented programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:javascript:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:javascript", + "key": "tldr", + "value": "high-level programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:php:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:php", + "key": "tldr", + "value": "scripting language focused on web development", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:python:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:python", + "key": "tldr", + "value": "general-purpose programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + }, + { + "id": "claim:programming-language:ruby:tldr", + "subjectId": "urn:ce:computing-basics:programming-language:ruby", + "key": "tldr", + "value": "general-purpose programming language", + "status": "asserted", + "evidenceRefs": [ + "src:wikidata:computing-basics" + ], + "observedAt": "2026-10-19" + } + ] +} diff --git a/examples/sources/wikidata/computing-basics.json b/examples/sources/wikidata/computing-basics.json new file mode 100644 index 0000000..aad61fb --- /dev/null +++ b/examples/sources/wikidata/computing-basics.json @@ -0,0 +1,625 @@ +{ + "format": "wikidata-entity-export", + "formatVersion": 1, + "retrievedAt": "2026-10-19", + "rightsNote": "Pinned local subset of Wikidata structured data. Wikidata structured data is dedicated to the public domain under CC0. This file is not a live query.", + "source": { + "id": "src:wikidata:computing-basics", + "uri": "https://www.wikidata.org/wiki/Wikidata:Licensing", + "title": "Wikidata", + "license": "CC0-1.0", + "attribution": "Wikidata contributors" + }, + "dataspace": { + "id": "computing-basics", + "title": "Computing basics", + "description": "A pinned CC0 subset of Wikidata items about programming languages, the people who designed them, and two formalisms.", + "heading": "Explore computing basics" + }, + "factPolicy": "show-all", + "acyclicPredicates": [ + "designed-by" + ], + "kinds": [ + { + "id": "programming-language", + "label": "Programming language", + "plural": "Programming languages" + }, + { + "id": "language-designer", + "label": "Language designer", + "plural": "Language designers" + }, + { + "id": "formalism", + "label": "Formalism", + "plural": "Formalisms" + } + ], + "relationLabels": { + "designed-by": { + "forward": "Designed by", + "inverse": "Designer of" + } + }, + "kindByQid": { + "Q28865": { + "kind": "programming-language", + "key": "python" + }, + "Q2005": { + "kind": "programming-language", + "key": "javascript" + }, + "Q15777": { + "kind": "programming-language", + "key": "c" + }, + "Q251": { + "kind": "programming-language", + "key": "java" + }, + "Q2407": { + "kind": "programming-language", + "key": "cpp" + }, + "Q161053": { + "kind": "programming-language", + "key": "ruby" + }, + "Q59": { + "kind": "programming-language", + "key": "php" + }, + "Q30942": { + "kind": "language-designer", + "key": "guido-van-rossum" + }, + "Q92648": { + "kind": "language-designer", + "key": "brendan-eich" + }, + "Q45575": { + "kind": "language-designer", + "key": "dennis-ritchie" + }, + "Q92622": { + "kind": "language-designer", + "key": "james-gosling" + }, + "Q92620": { + "kind": "language-designer", + "key": "bjarne-stroustrup" + }, + "Q92748": { + "kind": "language-designer", + "key": "yukihiro-matsumoto" + }, + "Q92768": { + "kind": "language-designer", + "key": "rasmus-lerdorf" + }, + "Q8366": { + "kind": "formalism", + "key": "algorithm" + }, + "Q9143": { + "kind": "formalism", + "key": "programming-language" + } + }, + "entities": { + "Q28865": { + "type": "item", + "id": "Q28865", + "labels": { + "en": { + "language": "en", + "value": "Python" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "general-purpose programming language" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Python programming language" + } + ] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q30942" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q2005": { + "type": "item", + "id": "Q2005", + "labels": { + "en": { + "language": "en", + "value": "JavaScript" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "high-level programming language" + } + }, + "aliases": { + "en": [] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q92648" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q15777": { + "type": "item", + "id": "Q15777", + "labels": { + "en": { + "language": "en", + "value": "C" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "general-purpose programming language" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "C programming language" + } + ] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q45575" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q251": { + "type": "item", + "id": "Q251", + "labels": { + "en": { + "language": "en", + "value": "Java" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "object-oriented programming language" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "java programming language" + } + ] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q92622" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q2407": { + "type": "item", + "id": "Q2407", + "labels": { + "en": { + "language": "en", + "value": "C++" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "general-purpose programming language" + } + }, + "aliases": { + "en": [] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q92620" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q161053": { + "type": "item", + "id": "Q161053", + "labels": { + "en": { + "language": "en", + "value": "Ruby" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "general-purpose programming language" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Ruby programming language" + } + ] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q92748" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q59": { + "type": "item", + "id": "Q59", + "labels": { + "en": { + "language": "en", + "value": "PHP" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "scripting language focused on web development" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "PHP programming language" + } + ] + }, + "claims": { + "P287": [ + { + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q92768" + } + } + }, + "type": "statement", + "rank": "normal" + } + ] + } + }, + "Q30942": { + "type": "item", + "id": "Q30942", + "labels": { + "en": { + "language": "en", + "value": "Guido van Rossum" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "Dutch programmer and creator of Python" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "gvanrossum" + } + ] + }, + "claims": {} + }, + "Q92648": { + "type": "item", + "id": "Q92648", + "labels": { + "en": { + "language": "en", + "value": "Brendan Eich" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "American computer programmer and technology executive" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Brendan O. Eich" + } + ] + }, + "claims": {} + }, + "Q45575": { + "type": "item", + "id": "Q45575", + "labels": { + "en": { + "language": "en", + "value": "Dennis M. Ritchie" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "American computer scientist, co-inventor of the Unix operating system and the C programming language (1941\u20132011)" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Dennis MacAlistair Ritchie" + } + ] + }, + "claims": {} + }, + "Q92622": { + "type": "item", + "id": "Q92622", + "labels": { + "en": { + "language": "en", + "value": "James Gosling" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "Canadian computer scientist" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "James Arthur Gosling" + } + ] + }, + "claims": {} + }, + "Q92620": { + "type": "item", + "id": "Q92620", + "labels": { + "en": { + "language": "en", + "value": "Bjarne Stroustrup" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "Danish computer scientist, creator of C++ (born 1950)" + } + }, + "aliases": { + "en": [] + }, + "claims": {} + }, + "Q92748": { + "type": "item", + "id": "Q92748", + "labels": { + "en": { + "language": "en", + "value": "Yukihiro Matsumoto" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "Japanese computer scientist who created the Ruby programming language" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Matz" + } + ] + }, + "claims": {} + }, + "Q92768": { + "type": "item", + "id": "Q92768", + "labels": { + "en": { + "language": "en", + "value": "Rasmus Lerdorf" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "Danish programmer and creator of PHP" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "Rasmus Jay Lerdorf" + } + ] + }, + "claims": {} + }, + "Q8366": { + "type": "item", + "id": "Q8366", + "labels": { + "en": { + "language": "en", + "value": "algorithm" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "sequence of instructions to perform a task" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "algorithms" + } + ] + }, + "claims": {} + }, + "Q9143": { + "type": "item", + "id": "Q9143", + "labels": { + "en": { + "language": "en", + "value": "programming language" + } + }, + "descriptions": { + "en": { + "language": "en", + "value": "language for communicating instructions to a machine" + } + }, + "aliases": { + "en": [ + { + "language": "en", + "value": "computer language" + } + ] + }, + "claims": {} + } + } +} diff --git a/tests/test_quality.py b/tests/test_quality.py index 7c28620..0c1b81e 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -76,7 +76,7 @@ class ExampleReports(unittest.TestCase): reports = report_dataspaces(EXAMPLES, today="2026-10-19") self.assertEqual( sorted(report["dataspace"] for report in reports), - ["software-architecture", "systems-thinking"], + ["computing-basics", "software-architecture", "systems-thinking"], ) for report in reports: self.assertGreater(report["provenance"]["records"], 0) diff --git a/tests/test_wikidata_connector.py b/tests/test_wikidata_connector.py new file mode 100644 index 0000000..aa8ca56 --- /dev/null +++ b/tests/test_wikidata_connector.py @@ -0,0 +1,150 @@ +"""Certification for the Wikidata export connector.""" + +from __future__ import annotations + +import ast +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from content_explorer.contract import validate_documents # noqa: E402 +from content_explorer.ingest import ALLOWED_LICENSES, load_json_records # noqa: E402 +from connectors.wikidata_export import CONNECTOR_VERSION, run # noqa: E402 + +PIN = ROOT / "examples" / "sources" / "wikidata" / "computing-basics.json" +PUBLISHED = ROOT / "examples" / "dataspaces" / "computing-basics" +KIND_NAMES = ("programming-language", "language-designer", "formalism") +UI_ROOTS = (ROOT / "ui" / "app", ROOT / "ui" / "components") + + +def _write(document: dict, directory: Path) -> Path: + directory.mkdir(parents=True, exist_ok=True) + path = directory / "export.json" + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +class WikidataConnector(unittest.TestCase): + def test_pinned_export_publishes_and_replays(self) -> None: + with self.assertRaises(ValueError): + load_json_records(PIN) + first = run(PIN) + second = run(PIN) + snapshot = first["snapshot"] + report = first["report"] + self.assertTrue(report["published"]) + self.assertEqual(report["connectorVersion"], f"wikidata-export/{CONNECTOR_VERSION}") + digest = __import__("hashlib").sha256(PIN.read_bytes()).hexdigest() + self.assertEqual(report["sourceDigest"], digest) + self.assertEqual(report["inputDigest"], digest) + self.assertEqual(report["counts"]["candidates"], 16) + self.assertEqual(report["counts"]["accepted"], len(snapshot["entities"])) + self.assertEqual( + [entity["id"] for entity in snapshot["entities"]], + [entity["id"] for entity in second["snapshot"]["entities"]], + ) + self.assertEqual(report["contentHash"], second["report"]["contentHash"]) + python = next(entity for entity in snapshot["entities"] if entity["id"].endswith(":python")) + self.assertTrue(python["id"].startswith("urn:ce:computing-basics:")) + self.assertIn("Q28865", python["aliases"]) + source = snapshot["sources"][0] + self.assertIn(source["license"], ALLOWED_LICENSES) + for claim in snapshot["claims"]: + self.assertEqual(claim["evidenceRefs"], [source["id"]]) + self.assertEqual(validate_documents(snapshot, first["manifest"], require_size=True), []) + committed = json.loads((PUBLISHED / "snapshot.json").read_text(encoding="utf-8")) + committed_manifest = json.loads((PUBLISHED / "manifest.json").read_text(encoding="utf-8")) + self.assertEqual(snapshot, committed) + self.assertEqual(first["manifest"], committed_manifest) + human = (PUBLISHED / "import-report.md").read_text(encoding="utf-8") + self.assertIn(f"Connector version: wikidata-export/{CONNECTOR_VERSION}", human) + self.assertIn(f"Source digest: {digest}", human) + + def test_forbidden_rights_unresolved_reference_and_identity_conflict_do_not_publish(self) -> None: + document = json.loads(PIN.read_text(encoding="utf-8")) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + rights = json.loads(json.dumps(document)) + rights["source"]["license"] = "CC-BY-NC-4.0" + rights_dest = root / "rights" + rights_result = run(_write(rights, root / "rights-src"), rights_dest) + self.assertFalse(rights_result["report"]["published"]) + self.assertIsNone(rights_result["snapshot"]) + self.assertEqual(rights_result["report"]["quarantined"][0]["reason"], "rights") + self.assertFalse((rights_dest / "snapshot.json").exists()) + + unresolved = json.loads(json.dumps(document)) + unresolved["entities"]["Q28865"]["claims"]["P287"].append({ + "mainsnak": { + "snaktype": "value", + "property": "P287", + "datavalue": {"type": "wikibase-entityid", "value": {"id": "Q1"}}, + }, + "type": "statement", + }) + unresolved_dest = root / "unresolved" + unresolved_result = run(_write(unresolved, root / "unresolved-src"), unresolved_dest) + self.assertFalse(unresolved_result["report"]["published"]) + self.assertIsNone(unresolved_result["snapshot"]) + self.assertTrue(unresolved_result["report"]["dangling"]) + self.assertFalse((unresolved_dest / "snapshot.json").exists()) + + conflict = json.loads(json.dumps(document)) + conflict["kindByQid"]["Q2005"]["key"] = conflict["kindByQid"]["Q28865"]["key"] + conflict["kindByQid"]["Q2005"]["kind"] = conflict["kindByQid"]["Q28865"]["kind"] + conflict_dest = root / "conflict" + conflict_result = run(_write(conflict, root / "conflict-src"), conflict_dest) + self.assertFalse(conflict_result["report"]["published"]) + self.assertIsNone(conflict_result["snapshot"]) + self.assertEqual(conflict_result["report"]["conflicts"][0]["reason"], "duplicate-identity") + self.assertFalse((conflict_dest / "snapshot.json").exists()) + + def test_changed_export_refreshes_the_snapshot(self) -> None: + document = json.loads(PIN.read_text(encoding="utf-8")) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + dest = root / "out" + first = run(PIN, dest) + document["entities"]["Q28865"]["labels"]["en"]["value"] = "Python language" + changed_path = root / "changed.json" + changed_path.write_text(json.dumps(document), encoding="utf-8") + changed = run(changed_path, dest) + self.assertTrue(changed["report"]["published"]) + self.assertNotEqual(changed["report"]["sourceDigest"], first["report"]["sourceDigest"]) + self.assertEqual(changed["report"]["connectorVersion"], first["report"]["connectorVersion"]) + self.assertGreater(changed["report"]["counts"]["accepted"], 0) + published = json.loads((dest / "snapshot.json").read_text(encoding="utf-8")) + python = next(entity for entity in published["entities"] if entity["id"].endswith(":python")) + self.assertEqual(python["title"], "Python language") + self.assertNotEqual( + [entity["title"] for entity in first["snapshot"]["entities"]], + [entity["title"] for entity in published["entities"]], + ) + + def test_connector_does_not_import_ui_and_components_omit_kind_names(self) -> None: + source = (ROOT / "connectors" / "wikidata_export.py").read_text(encoding="utf-8") + tree = ast.parse(source) + imported: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.append(node.module or "") + self.assertFalse(any(name == "react" or name.startswith("ui") for name in imported)) + blob = "\n".join( + path.read_text(encoding="utf-8") + for root in UI_ROOTS + for path in root.rglob("*") + if path.suffix in {".ts", ".tsx"} + ) + for name in KIND_NAMES: + self.assertNotIn(name, blob) + + +if __name__ == "__main__": + unittest.main() diff --git a/ui/components/both-dataspaces.test.tsx b/ui/components/both-dataspaces.test.tsx index a72c23a..e10f255 100644 --- a/ui/components/both-dataspaces.test.tsx +++ b/ui/components/both-dataspaces.test.tsx @@ -48,7 +48,8 @@ function buttonNamed(label: string): HTMLButtonElement { describe("both dataspaces", () => { it("browses, searches, filters, sorts, and opens a detail for each field", async () => { - expect(listNavigation().map((item) => item.id).sort()).toEqual([...SPACES].sort()); + const ids = listNavigation().map((item) => item.id); + for (const id of SPACES) expect(ids).toContain(id); for (const id of SPACES) { const space = findDataspace(id); diff --git a/ui/lib/computing-basics.test.ts b/ui/lib/computing-basics.test.ts new file mode 100644 index 0000000..c20fba1 --- /dev/null +++ b/ui/lib/computing-basics.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { entitySlug, findDataspace, findEntity, findKind } from "./catalog"; + +const directory = join(process.cwd(), "../examples/dataspaces/computing-basics"); + +describe("computing-basics catalog", () => { + it("reads kind labels and an entity title from the published dataspace", () => { + const snapshot = JSON.parse(readFileSync(join(directory, "snapshot.json"), "utf8")) as { + dataspace: { id: string; title: string }; + kinds: { id: string; label: string }[]; + entities: { id: string; kindId: string; title: string }[]; + sources: { id: string; title: string; license: string }[]; + }; + const manifest = JSON.parse(readFileSync(join(directory, "manifest.json"), "utf8")) as { + kindViews: Record; + home: { featuredKinds: string[] }; + }; + const kindId = manifest.home.featuredKinds[0]; + const declared = snapshot.kinds.find((kind) => kind.id === kindId); + const kind = findKind(snapshot.dataspace.id, kindId); + expect(declared).toBeTruthy(); + expect(kind?.label).toBe(declared?.label); + expect(kind?.columns.map((column) => column.key)).toEqual(manifest.kindViews[kindId].columns); + + const entity = snapshot.entities.find((item) => item.kindId === kindId); + expect(entity).toBeTruthy(); + const view = findEntity(snapshot.dataspace.id, kindId, entitySlug(entity!.id, kindId)); + const source = snapshot.sources[0]; + expect(view?.title).toBe(entity?.title); + expect( + view?.sources.some((item) => item.title === source.title || item.license === source.license), + ).toBe(true); + expect(findDataspace(snapshot.dataspace.id)?.title).toBe(snapshot.dataspace.title); + }); +}); diff --git a/workplans/CONTENT-WP-0006-external-pilot.md b/workplans/CONTENT-WP-0006-external-pilot.md index 5f7fdf9..a7e1e42 100644 --- a/workplans/CONTENT-WP-0006-external-pilot.md +++ b/workplans/CONTENT-WP-0006-external-pilot.md @@ -9,7 +9,7 @@ flavor: implementation depends_on: - CONTENT-WP-0005 created: "2026-10-12" -updated: "2026-10-12" +updated: "2026-10-19" state_hub_workstream_id: "b1767962-549f-5de2-9503-a997a21a2133" --- @@ -17,13 +17,15 @@ state_hub_workstream_id: "b1767962-549f-5de2-9503-a997a21a2133" Attach an independently maintained knowledge source and validate user value. +The Wikidata export connector, its fixture certification, and the static-projection no-go are recorded in `connectors/wikidata_export.py`, `tests/test_wikidata_connector.py`, and `workplans/evidence/CONTENT-WP-0006-boundaries.md`. The real-user pilot and the observed-need tasks stay open. This workplan is not finished. + ## Work packages ### Implement a third connector, GitHub issues or Wikidata, after a rights and rate review. ```task id: CONTENT-WP-0006-T01 -status: todo +status: done priority: high state_hub_task_id: "ae191b49-108f-5c80-9964-0e0f3fb95b79" ``` @@ -32,7 +34,7 @@ state_hub_task_id: "ae191b49-108f-5c80-9964-0e0f3fb95b79" ```task id: CONTENT-WP-0006-T02 -status: todo +status: done priority: high state_hub_task_id: "a6f752d7-788a-54b7-8bd4-8a0f8889a6e4" ``` @@ -70,7 +72,7 @@ state_hub_task_id: "cf85f981-f685-5b61-99f1-bb25d1d98dc2" ```task id: CONTENT-WP-0006-T06 -status: todo +status: done priority: high state_hub_task_id: "81bafca2-e037-5c65-9d08-a0404c6d6aa1" ``` @@ -79,7 +81,7 @@ state_hub_task_id: "81bafca2-e037-5c65-9d08-a0404c6d6aa1" ```task id: CONTENT-WP-0006-T07 -status: todo +status: done priority: high state_hub_task_id: "3259fad2-e2fa-54c9-9fd4-b2deb0951850" ``` @@ -88,7 +90,7 @@ state_hub_task_id: "3259fad2-e2fa-54c9-9fd4-b2deb0951850" ```task id: CONTENT-WP-0006-T08 -status: todo +status: done priority: high state_hub_task_id: "ebed1e8e-8b05-58e2-90ea-d3afcb2d7e72" ``` diff --git a/workplans/evidence/CONTENT-WP-0006-boundaries.md b/workplans/evidence/CONTENT-WP-0006-boundaries.md new file mode 100644 index 0000000..bf4ff26 --- /dev/null +++ b/workplans/evidence/CONTENT-WP-0006-boundaries.md @@ -0,0 +1,23 @@ +# CONTENT-WP-0006 boundaries for this increment + +This note records the direction taken for the third connector and the boundaries this increment does not cross. It is not a user interview and it is not a pilot transcript. + +## Source + +Wikidata is the connector that landed. The pinned input is `examples/sources/wikidata/computing-basics.json`, mapped by `connectors/wikidata_export.py`. Wikidata structured data is CC0, and `CC0-1.0` is already an allowed publish license in `content_explorer/ingest.py` (`ALLOWED_LICENSES`) and `contracts/fact-policy.md` requires a source on every relation. GitHub issues were not implemented. Issue text is not CC0, CC-BY-4.0, or MIT without a per-repository rights review, and this increment does not call a live API. `architecture/ConnectorContract.md` names "GitHub or Wikidata" as the third connector and requires a license review before activation. `INTENT.md` lists that third connector as a subsequent integration target after the local JSON and Markdown loaders. + +## Real-user pilot + +A pilot with real users is not done. `workplans/CONTENT-WP-0006-external-pilot.md` keeps CONTENT-WP-0006-T03 open for that pilot. This note is the stand-in for a harness that cannot hold those sessions. + +## API, JSON-LD, MCP, and Ask + +Those interfaces are not built. `INTENT.md` principle 8 says static and exportable graph and HTTP/JSON come first, and MCP and richer interfaces come later. CONTENT-WP-0006-T04 stays open until needs are observed from the pilot. `INTENT.md` also lists a production LLM answer service as a v0.1 non-goal. + +## Dynamic deployment and search + +Dynamic deployment and a search backend stay a no-go. `decisions/ADR-0002-static-projection-first.md` decides to begin with offline connectors and a versioned static snapshot, with filter-in-browser at small scale, and to add remote read APIs, indexed search, and graph storage only with measured limits. CONTENT-WP-0006-T08 is the gate for that go or no-go. This increment follows the no-go. + +## OnCo authorization + +CONTENT-WP-0007 stays blocked until a human OnCo data-use authorization exists. `workplans/CONTENT-WP-0007-onco-reference-slice.md` sets that workplan to `blocked`, and CONTENT-WP-0007-T01 waits on `OnCo LICENSE-DATA` before any real reference-slice import. This increment does not read `upstream-onco/` data.