coulomb/prj-forgejo-org-refactor
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix access inventory evidence and reconcile closure coordination

Browse source 
Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a11beb-4eef-7fc1-94a7-597130a01ae9
This commit is contained in:
tegwick 2026-10-09 00:23:04 +02:00
parent 490a89dca6
commit 57ad4abefc
9 changed files with 342 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

100
tools/access_consumers.py
View file

@ -17,9 +17,12 @@ import csv
import json
import os
import re
import stat
import subprocess
from pathlib import Path
from inventory import BINARY, SENSITIVE
FORGE_HOSTS = ("forgejo.coulomb.social", "gitea.coulomb.social")
URL = re.compile(
r"(?:https?://)?(?P<host>forgejo\.coulomb\.social|gitea\.coulomb\.social)"
@ -34,6 +37,7 @@ KINDS = [
("record", lambda f, p, line: f.startswith(("docs/evidence/", "evidence/", "tests/", "test/"))
or "/evidence/" in f or "/tests/" in f or f.endswith(("PKG-INFO", ".sum"))),
("raw-fetch", lambda f, p, line: "/raw/" in p or "/media/" in p),
("package-consumer", lambda f, p, line: p.startswith(("/api/packages/", "/api/v1/packages/"))),
("api-client", lambda f, p, line: p.startswith("/api/")),
("ci-action", lambda f, p, line: ".forgejo/workflows" in f or ".gitea/workflows" in f
or ".github/workflows" in f),
@ -65,29 +69,68 @@ def target_repo(path):
parts = [p for p in (path or "").split("/") if p]
if parts[:2] == ["api", "v1"] and len(parts) >= 5 and parts[2] == "repos":
parts = parts[3:]
elif parts and parts[0] in {"api", "v2"}:
elif parts and parts[0] == "api":
# Package paths name an owner/index, not a repository. Other API
# endpoints also cannot establish a repository identity.
return None
elif parts and parts[0] == "v2":
parts = parts[1:]
if len(parts) >= 2 and parts[0] not in {"user", "explore", "assets"}:
return f"{parts[0]}/{parts[1].removesuffix('.git').split('@')[0].split(':')[0]}"
return None
def scan_repo(repo):
def scan_repo(repo, diagnostics=None):
hits = []
for dirpath, dirnames, filenames in os.walk(repo):
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and not d.startswith(".")
or d in {".forgejo", ".gitea", ".github"}]
for name in filenames:
if name in SKIP_FILES:
continue
diagnostics = diagnostics if diagnostics is not None else []
def excluded(path, reason):
diagnostics.append({"repository": repo.name, "path": path, "reason": reason})
def walk_error(error):
excluded(str(Path(error.filename).relative_to(repo)), "unreadable-directory")
for dirpath, dirnames, filenames in os.walk(repo, onerror=walk_error):
kept = []
for d in sorted(dirnames):
rel = str((Path(dirpath) / d).relative_to(repo))
if (Path(dirpath) / d).is_symlink():
excluded(rel, "symlink-not-followed")
elif d in SKIP_DIRS or (d.startswith(".") and d not in {".forgejo", ".gitea", ".github"}):
excluded(rel, "excluded-directory")
else:
kept.append(d)
dirnames[:] = kept
for name in sorted(filenames):
full = Path(dirpath) / name
rel = str(full.relative_to(repo))
if name in SKIP_FILES:
excluded(rel, "generated-work-record")
continue
if SENSITIVE.search(rel):
excluded(rel, "sensitive-or-encrypted")
continue
if full.suffix.lower() in BINARY:
excluded(rel, "binary-extension")
continue
try:
if full.is_symlink() or full.stat().st_size > MAX_BYTES:
if full.is_symlink():
excluded(rel, "symlink-not-followed")
continue
info = full.stat()
if not stat.S_ISREG(info.st_mode):
excluded(rel, "non-regular-file")
continue
if info.st_size > MAX_BYTES:
excluded(rel, "over-1MB")
continue
text = full.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
except UnicodeDecodeError:
excluded(rel, "non-utf8-content")
continue
except OSError:
excluded(rel, "unreadable-file")
continue
if "\0" in text:
excluded(rel, "binary-content")
continue
rel = str(full.relative_to(repo))
for number, line in enumerate(text.splitlines(), 1):
for m in URL.finditer(line):
hits.append({
@ -119,7 +162,7 @@ def runtime_consumers(runtime):
m = URL.search(u)
if m:
out.append({"consumer": f"argocd:{app['name']}", "class": "cluster-gitops",
"kind": "gitops-source", "target": target_repo(m.group("path")),
"host": m.group("host"), "kind": "gitops-source", "target": target_repo(m.group("path")),
"credential": "unknown (ArgoCD repo credentials not inspected)"})
for w in runtime.get("workloads", []):
for image in w.get("images", []):
@ -128,9 +171,10 @@ def runtime_consumers(runtime):
secrets = w.get("image_pull_secret_names") or []
out.append({"consumer": f"k3s:{w['namespace']}/{w['kind']}/{w['name']}",
"class": "cluster-image-pull", "kind": "image-pull",
"host": m.group("host"),
"target": target_repo(m.group("path")),
"credential": "pull secret named: " + ",".join(secrets)
if secrets else "anonymous (no imagePullSecrets)"})
if secrets else "unknown (no imagePullSecrets; node registry auth not inspected)"})
return out
@ -160,17 +204,19 @@ def main():
consumers = runtime_consumers(runtime)
remotes = collections.Counter()
scanned = []
diagnostics = []
coordinator = Path(__file__).resolve().parents[1]
for repo in sorted(Path(args.root).iterdir()):
if not (repo / ".git").exists() or repo.name.startswith("rescued-"):
if not (repo / ".git").exists() or repo.name.startswith("rescued-") or repo.resolve() == coordinator:
continue
scanned.append(repo.name)
remotes[remote_form(repo) or "none"] += 1
grouped = collections.defaultdict(list)
for h in scan_repo(repo):
grouped[(h["kind"], h["target"])].append(f"{h['file']}:{h['line']}")
for (kind, target), where in sorted(grouped.items(), key=lambda kv: (kv[0][0], str(kv[0][1]))):
for h in scan_repo(repo, diagnostics):
grouped[(h["host"], h["kind"], h["target"])].append(f"{h['file']}:{h['line']}")
for (host, kind, target), where in sorted(grouped.items(), key=lambda kv: (kv[0][0], kv[0][1], str(kv[0][2]))):
consumers.append({"consumer": f"repo:{repo.name}", "class": CLASS_OF_KIND[kind],
"kind": kind, "target": target, "locations": where[:10],
"host": host, "kind": kind, "target": target, "locations": where[:10],
"location_count": len(where)})
for c in consumers:
@ -179,19 +225,28 @@ def main():
summary = collections.Counter(c["class"] for c in consumers)
report = {
"schema_version": "0.1",
"schema_version": "0.2",
"workplan": "ORGREF-WP-0002-T02",
"runtime_capture": os.path.basename(args.runtime),
"scanned_repositories": len(scanned),
"local_remote_forms": dict(remotes),
"summary_by_class": dict(sorted(summary.items())),
"anonymous_image_pulls": sum(1 for c in consumers if c["class"] == "cluster-image-pull"
and c["credential"].startswith("anonymous")),
"image_pulls_without_named_credentials": sum(1 for c in consumers if c["class"] == "cluster-image-pull"
and c["credential"].startswith("unknown")),
"consumers": consumers,
"scan_exclusions": diagnostics,
"scan_error_count": sum(d["reason"].startswith("unreadable") for d in diagnostics),
"limits": [
"Runtime classes come from the runtime capture named above, not a live read.",
"ArgoCD repository credentials and pull-secret contents were not inspected; "
"a named pull secret is not proof that it authenticates to the forge.",
"Missing imagePullSecrets do not prove anonymous pulls: node-level registry "
"auth and service-account defaults require owner evidence.",
"Sensitive/encrypted, binary, large, symlinked and excluded paths are recorded "
"in scan_exclusions; unreadable paths prevent a completeness claim.",
"The coordinator is excluded to prevent generated inventory self-reference.",
"Hostnames are preserved so legacy-host references can be reviewed separately.",
"Package indexes name a namespace, not a verified repository; their target is null.",
"Only literal forge hostnames are found; computed URLs and consumers outside "
"these working trees (external sites, Telegram, other people's clones) are not.",
"history/ and archived/ directories are skipped; they record, they do not read.",
@ -199,7 +254,8 @@ def main():
}
Path(args.output).write_text(json.dumps(report, indent=1, sort_keys=False) + "\n")
print(json.dumps({k: report[k] for k in ("scanned_repositories", "local_remote_forms",
"summary_by_class", "anonymous_image_pulls")},
"summary_by_class", "image_pulls_without_named_credentials",
"scan_error_count")},
indent=1))