Implement source-backed NetKingdom agent security knowledge pilot
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a101af-e240-7882-a1e3-e37720cbdbea
This commit is contained in:
parent
caa7da2397
commit
a835c11ae7
54 changed files with 3634 additions and 27 deletions
255
src/emergent_security/knowledge.py
Normal file
255
src/emergent_security/knowledge.py
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
"""Read-only orientation retrieval. No models, credentials, grants, or live effects.
|
||||
|
||||
Only reviewed estate-general cards are served. External documents are hashed,
|
||||
never returned. Checksums detect drift, not authenticity: trust the release via
|
||||
the repository/release channel, not a digest supplied by an untrusted party.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import date
|
||||
import hashlib
|
||||
from importlib.resources import files
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class KnowledgeError(ValueError):
|
||||
"""Safe error: never include file contents, user context, or subprocess output."""
|
||||
|
||||
|
||||
class UniqueLoader(yaml.SafeLoader):
|
||||
pass
|
||||
|
||||
|
||||
def _mapping(loader, node, deep=False):
|
||||
loader.flatten_mapping(node)
|
||||
result = {}
|
||||
for key_node, value_node in node.value:
|
||||
key = loader.construct_object(key_node, deep=deep)
|
||||
if not isinstance(key, str) or key in result:
|
||||
raise KnowledgeError("invalid or duplicate YAML key")
|
||||
result[key] = loader.construct_object(value_node, deep=deep)
|
||||
return result
|
||||
|
||||
|
||||
UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _mapping)
|
||||
|
||||
|
||||
def read_yaml(path: Path) -> dict:
|
||||
try:
|
||||
value = yaml.load(path.read_text(encoding="utf-8"), Loader=UniqueLoader)
|
||||
except (OSError, UnicodeError, yaml.YAMLError):
|
||||
raise KnowledgeError("unreadable YAML document") from None
|
||||
if not isinstance(value, dict):
|
||||
raise KnowledgeError("expected YAML mapping")
|
||||
return value
|
||||
|
||||
|
||||
def confined(root: Path, name: str) -> Path:
|
||||
if not isinstance(name, str) or not name or Path(name).is_absolute() or ".." in Path(name).parts:
|
||||
raise KnowledgeError("invalid relative path")
|
||||
result = (root / name).resolve()
|
||||
if not result.is_relative_to(root.resolve()):
|
||||
raise KnowledgeError("path escapes declared root")
|
||||
return result
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
try:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
except OSError:
|
||||
raise KnowledgeError("unreadable content") from None
|
||||
|
||||
|
||||
def _index(rows: list, kind: str) -> dict:
|
||||
if not isinstance(rows, list):
|
||||
raise KnowledgeError(f"invalid {kind} list")
|
||||
result = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or not re.fullmatch(r"[a-z][a-z0-9-]*", str(row.get("id", ""))):
|
||||
raise KnowledgeError(f"invalid {kind} identifier")
|
||||
if row["id"] in result:
|
||||
raise KnowledgeError(f"duplicate {kind} identifier")
|
||||
result[row["id"]] = row
|
||||
return result
|
||||
|
||||
|
||||
def default_bundle() -> Path:
|
||||
try:
|
||||
return Path(str(files("emergent_security_knowledge")))
|
||||
except ModuleNotFoundError:
|
||||
# Source checkout use without installation; wheels use package resources.
|
||||
return Path(__file__).resolve().parents[2] / "knowledge"
|
||||
|
||||
|
||||
class Knowledge:
|
||||
def __init__(self, root: Path | None = None):
|
||||
self.root = (root or default_bundle()).resolve()
|
||||
self.release = read_yaml(confined(self.root, "release.yaml"))
|
||||
if self.release.get("schema") != "emergent.release.v1":
|
||||
raise KnowledgeError("unsupported release schema")
|
||||
hashes = self.release.get("files")
|
||||
if not isinstance(hashes, dict) or not {"catalog.yaml", "sources.yaml"} <= hashes.keys():
|
||||
raise KnowledgeError("incomplete release manifest")
|
||||
for name, expected in hashes.items():
|
||||
if not isinstance(expected, str) or not re.fullmatch(r"[a-f0-9]{64}", expected):
|
||||
raise KnowledgeError("invalid release digest")
|
||||
if digest(confined(self.root, name)) != expected:
|
||||
raise KnowledgeError("release content changed; review and reseal")
|
||||
self.catalog = read_yaml(confined(self.root, "catalog.yaml"))
|
||||
sources = read_yaml(confined(self.root, "sources.yaml"))
|
||||
if self.catalog.get("schema") != "emergent.knowledge.v1" or sources.get("schema") != "emergent.sources.v1":
|
||||
raise KnowledgeError("unsupported knowledge schema")
|
||||
if self.catalog.get("version") != self.release.get("version"):
|
||||
raise KnowledgeError("release version mismatch")
|
||||
if (not isinstance(self.catalog.get("startup_max_bytes"), int)
|
||||
or not 0 < self.catalog["startup_max_bytes"] <= 10000):
|
||||
raise KnowledgeError("invalid startup budget")
|
||||
self.cards = _index(self.catalog.get("cards"), "card")
|
||||
self.sources = _index(sources.get("sources"), "source")
|
||||
for card in self.cards.values():
|
||||
if (not isinstance(card.get("path"), str) or card["path"] not in hashes
|
||||
or not isinstance(card.get("sources"), list) or not card["sources"]):
|
||||
raise KnowledgeError("unsealed card or missing provenance")
|
||||
if any(not isinstance(ref, str) or ref not in self.sources for ref in card["sources"]):
|
||||
raise KnowledgeError("unknown source reference")
|
||||
if not isinstance(card.get("audiences"), list) or not all(isinstance(v, str) for v in card["audiences"]):
|
||||
raise KnowledgeError("invalid audiences")
|
||||
for source in self.sources.values():
|
||||
if source.get("status") not in {"active", "withdrawn", "superseded"}:
|
||||
raise KnowledgeError("invalid source lifecycle")
|
||||
if source.get("scope") != "estate-general":
|
||||
raise KnowledgeError("source classification not supported")
|
||||
if not isinstance(source.get("documents"), list) or not source["documents"]:
|
||||
raise KnowledgeError("missing source documents")
|
||||
if not re.fullmatch(r"[a-f0-9]{40,64}", str(source.get("revision", ""))):
|
||||
raise KnowledgeError("invalid source revision")
|
||||
if not re.fullmatch(r"[a-z][a-z0-9-]*", str(source.get("repository", ""))):
|
||||
raise KnowledgeError("invalid source repository")
|
||||
try:
|
||||
verified = date.fromisoformat(source["last_verified"])
|
||||
review = date.fromisoformat(source["review_after"])
|
||||
if review <= verified:
|
||||
raise ValueError
|
||||
except (ValueError, TypeError, KeyError):
|
||||
raise KnowledgeError("invalid review dates") from None
|
||||
for document in source["documents"]:
|
||||
if not isinstance(document, dict):
|
||||
raise KnowledgeError("invalid source document")
|
||||
confined(Path("/source"), document.get("path"))
|
||||
if not re.fullmatch(r"[a-f0-9]{64}", str(document.get("sha256", ""))):
|
||||
raise KnowledgeError("invalid source digest")
|
||||
if not str(document.get("url", "")).startswith("https://forgejo.coulomb.social/"):
|
||||
raise KnowledgeError("invalid canonical source URL")
|
||||
self.package_digest = digest(self.root / "release.yaml")
|
||||
|
||||
def source_status(self, ref: str, repositories: Path | None, today: date) -> str:
|
||||
source = self.sources[ref]
|
||||
if source["status"] != "active":
|
||||
return source["status"]
|
||||
if source.get("superseded_by"):
|
||||
return "superseded"
|
||||
if today < date.fromisoformat(source["last_verified"]):
|
||||
return "future-verification"
|
||||
if today >= date.fromisoformat(source["review_after"]):
|
||||
return "review-due"
|
||||
if repositories is None:
|
||||
return "offline"
|
||||
try:
|
||||
repo = confined(repositories, source["repository"])
|
||||
for document in source["documents"]:
|
||||
path = confined(repo, document["path"])
|
||||
if not path.is_file():
|
||||
return "unavailable"
|
||||
if digest(path) != document["sha256"]:
|
||||
return "changed"
|
||||
except KnowledgeError:
|
||||
return "unavailable"
|
||||
return "verified-files"
|
||||
|
||||
def retrieve(self, card_id: str, *, audience: str = "local",
|
||||
repositories: Path | None = None, today: date | None = None) -> dict:
|
||||
if card_id not in self.cards:
|
||||
raise KnowledgeError("unknown card; use list")
|
||||
card = self.cards[card_id]
|
||||
# No caller-supplied tenant label can authorize private retrieval.
|
||||
if card.get("classification") != "estate-general" or card.get("tenant") is not None:
|
||||
raise KnowledgeError("private or tenant content is not served")
|
||||
if audience not in card.get("audiences", []):
|
||||
raise KnowledgeError("audience not approved for this card")
|
||||
statuses = {ref: self.source_status(ref, repositories, today or date.today())
|
||||
for ref in card["sources"]}
|
||||
withdrawn = any(value in {"withdrawn", "superseded"} for value in statuses.values())
|
||||
stale = any(value in {"changed", "review-due", "future-verification"} for value in statuses.values())
|
||||
# Withdrawn/changed guidance is withheld; an unavailable source leaves a
|
||||
# dated snapshot useful for orientation, with no current-authority claim.
|
||||
state = "withheld" if withdrawn or stale else (
|
||||
"verified-files" if all(v == "verified-files" for v in statuses.values()) else "offline-snapshot")
|
||||
text = "" if state == "withheld" else confined(self.root, card["path"]).read_text(encoding="utf-8")
|
||||
if card_id == "start" and len(text.encode()) > self.catalog["startup_max_bytes"]:
|
||||
raise KnowledgeError("startup context budget exceeded")
|
||||
return {
|
||||
"schema": "emergent.knowledge-receipt.v1",
|
||||
"package_version": self.catalog["version"], "package_digest": self.package_digest,
|
||||
"release_status": self.catalog["status"], "card": card_id, "audience": audience,
|
||||
"status": state, "authority": "orientation-only", "live_state_verified": False,
|
||||
"scope_resolution": "caller-must-resolve; no task context collected",
|
||||
"sources": [{"id": ref, "status": status,
|
||||
"revision": self.sources[ref]["revision"],
|
||||
"references": [d["url"] for d in self.sources[ref]["documents"]]}
|
||||
for ref, status in statuses.items()],
|
||||
"content_bytes": len(text.encode()), "content": text,
|
||||
}
|
||||
|
||||
def check(self, repositories: Path | None = None, today: date | None = None) -> dict:
|
||||
statuses = {ref: self.source_status(ref, repositories, today or date.today()) for ref in self.sources}
|
||||
return {"schema": "emergent.knowledge-check.v1", "version": self.catalog["version"],
|
||||
"package_digest": self.package_digest, "sources": statuses,
|
||||
"live_state_verified": False,
|
||||
"ok": all(s == "verified-files" for s in statuses.values())}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--bundle", type=Path, help="Explicit trusted release directory")
|
||||
parser.add_argument("--repositories", type=Path, help="Read-only source checkout parent; omit for offline")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("list")
|
||||
sub.add_parser("check")
|
||||
show = sub.add_parser("show")
|
||||
show.add_argument("card", nargs="?", default="start")
|
||||
show.add_argument("--audience", choices=["local", "codex", "claude", "grok", "generic"], default="local")
|
||||
show.add_argument("--json", action="store_true")
|
||||
try:
|
||||
args = parser.parse_args(argv)
|
||||
package = Knowledge(args.bundle)
|
||||
if args.command == "list":
|
||||
print("\n".join(package.cards))
|
||||
return 0
|
||||
if args.command == "check":
|
||||
result = package.check(args.repositories)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
result = package.retrieve(args.card, audience=args.audience, repositories=args.repositories)
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
else:
|
||||
print(f"Knowledge {result['package_version']} / {result['card']} / {result['status']} / orientation-only")
|
||||
print(f"Package digest: {result['package_digest']}")
|
||||
print(result["content"] or "Guidance withheld: source changed, withdrawn, superseded or review due.")
|
||||
print("Sources:")
|
||||
for source in result["sources"]:
|
||||
print(f"- {source['id']} ({source['status']}): {source['references'][0]}")
|
||||
return 2 if result["status"] == "withheld" else 0
|
||||
except KnowledgeError as error:
|
||||
print(f"knowledge: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue