Prepare org reference inventory and stewardship assignment review
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e782-9f45-7382-afff-77ad665b88ec
This commit is contained in:
parent
25dd4626e0
commit
660c429838
19 changed files with 54017 additions and 19 deletions
206
tools/inventory.py
Normal file
206
tools/inventory.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Read-only org-reference inventory. Emits reference tokens, never source lines."""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import io
|
||||
import zipfile
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
BASE = 'https://forgejo.coulomb.social'
|
||||
# Capture only the owner and first path component; never URL credentials/query.
|
||||
REF = re.compile(r'(?<![A-Za-z0-9_-])coulomb/([A-Za-z0-9_.-]+)')
|
||||
SENSITIVE = re.compile(r'(^|/)(\.env($|\.)|id_rsa|id_ed25519)|\.(pem|key|p12|pfx|age)$|sops\.|\.sops\.', re.I)
|
||||
BINARY = {'.png','.jpg','.jpeg','.gif','.pdf','.zip','.gz','.woff','.woff2','.ico','.mp4','.sqlite','.db'}
|
||||
SELF = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def run(repo, *args):
|
||||
r = subprocess.run(['git', '-C', str(repo), *args], capture_output=True)
|
||||
if r.returncode:
|
||||
raise RuntimeError('git ' + args[0] + ' failed')
|
||||
return r.stdout
|
||||
|
||||
|
||||
def get(url):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as r:
|
||||
return json.load(r)
|
||||
except urllib.error.HTTPError as e:
|
||||
raise RuntimeError('HTTP ' + str(e.code)) from None
|
||||
|
||||
|
||||
def pages(path):
|
||||
rows = []
|
||||
seen = set()
|
||||
for page in range(1, 1001):
|
||||
batch = get(BASE + path + f'?limit=50&page={page}')
|
||||
if not isinstance(batch, list):
|
||||
raise RuntimeError('expected paginated list')
|
||||
if not batch:
|
||||
return rows
|
||||
digest = hashlib.sha256(json.dumps(batch, sort_keys=True).encode()).hexdigest()
|
||||
if digest in seen:
|
||||
raise RuntimeError('pagination repeated a page')
|
||||
seen.add(digest)
|
||||
rows.extend(batch)
|
||||
raise RuntimeError('pagination limit exceeded')
|
||||
|
||||
|
||||
def categories(path):
|
||||
p = path.lower()
|
||||
kinds = []
|
||||
for flag, name in [
|
||||
('argocd' in p, 'gitops'),
|
||||
(p.endswith('chart.yaml'), 'chart-metadata'),
|
||||
('chart' in p or 'values' in p, 'chart-values'),
|
||||
(p.endswith('.gitmodules'), 'submodule'),
|
||||
(any(x in p for x in ['.gitea/','.forgejo/','.github/','workflow','jenkins','ci/']), 'ci'),
|
||||
(any(x in p for x in ['retention','prune','live-image','release','rollback']), 'release-retention'),
|
||||
(any(x in p for x in ['credential','policy','routing','access']), 'access-policy'),
|
||||
(any(x in p for x in ['lock','requirements','pyproject','package.json','npmrc']), 'package-consumer'),
|
||||
(any(x in p.split('/') for x in ['history','archived','evidence','workplans']), 'historical-or-planning'),
|
||||
]:
|
||||
if flag:
|
||||
kinds.append(name)
|
||||
return kinds or ['other']
|
||||
|
||||
|
||||
def scan(path):
|
||||
errors, exclusions, refs = [], [], []
|
||||
start = run(path, 'rev-parse', 'HEAD').decode().strip()
|
||||
remotes = []
|
||||
raw = run(path, 'config', '--get-regexp', r'^remote\..*\.url$') if run(path,'remote').strip() else b''
|
||||
for line in raw.decode(errors='replace').splitlines():
|
||||
key, value = line.split(' ', 1)
|
||||
match = REF.search(value)
|
||||
remotes.append({'remote':key.split('.')[1], 'owner_path':match.group(0).removesuffix('.git') if match else None,
|
||||
'transport':'https' if value.startswith('https:') else 'ssh-or-other'})
|
||||
files = sorted(set(run(path, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split(b'\0')) - {b''})
|
||||
h = hashlib.sha256()
|
||||
for encoded in files:
|
||||
name = encoded.decode(errors='replace')
|
||||
file = path / name
|
||||
if file.is_dir():
|
||||
exclusions.append({'path':name,'reason':'gitlink-or-directory-not-scanned'}); continue
|
||||
if file.is_symlink():
|
||||
exclusions.append({'path':name,'reason':'symlink-not-followed'}); continue
|
||||
if SENSITIVE.search(name):
|
||||
exclusions.append({'path':name,'reason':'sensitive-or-encrypted'}); continue
|
||||
if file.suffix.lower() in BINARY:
|
||||
exclusions.append({'path':name,'reason':'binary-extension'}); continue
|
||||
try:
|
||||
if file.stat().st_size > 4_000_000:
|
||||
exclusions.append({'path':name,'reason':'over-4MB'}); continue
|
||||
data = file.read_bytes()
|
||||
except OSError:
|
||||
errors.append({'path':name,'reason':'unreadable-or-missing'}); continue
|
||||
h.update(encoded + b'\0' + hashlib.sha256(data).digest())
|
||||
if b'\0' in data:
|
||||
exclusions.append({'path':name,'reason':'binary-content'}); continue
|
||||
for number, line in enumerate(data.decode(errors='replace').splitlines(), 1):
|
||||
for match in REF.finditer(line):
|
||||
refs.append({'path':name,'line':number,'reference':match.group(0), 'categories':categories(name)})
|
||||
end = run(path, 'rev-parse', 'HEAD').decode().strip()
|
||||
if start != end:
|
||||
errors.append({'reason':'HEAD-changed-during-scan'})
|
||||
return {'name':path.name, 'path':str(path), 'head':start,
|
||||
'content_sha256':h.hexdigest(), 'remotes':remotes, 'files_considered':len(files),
|
||||
'references':refs,'exclusions':exclusions,'errors':errors}
|
||||
|
||||
|
||||
|
||||
def scan_archive(repo):
|
||||
"""Read a public default-branch archive in memory, never extract to disk."""
|
||||
name=repo['name']
|
||||
branch=repo['default_branch']
|
||||
from urllib.parse import quote
|
||||
branch_info=get(BASE+'/api/v1/repos/coulomb/'+name+'/branches/'+quote(branch,safe=''))
|
||||
commit=branch_info['commit']['id']
|
||||
url=BASE+'/api/v1/repos/coulomb/'+name+'/archive/'+commit+'.zip'
|
||||
with urllib.request.urlopen(url,timeout=30) as response:
|
||||
raw=response.read(50_000_001)
|
||||
if len(raw)>50_000_000:
|
||||
raise RuntimeError('archive exceeds 50MB')
|
||||
refs=[];exclusions=[]
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as archive:
|
||||
for item in sorted(archive.infolist(),key=lambda i:i.filename):
|
||||
if item.is_dir(): continue
|
||||
path=item.filename.split('/',1)[-1]
|
||||
if SENSITIVE.search(path) or Path(path).suffix.lower() in BINARY or item.file_size>4_000_000:
|
||||
exclusions.append({'path':path,'reason':'sensitive-binary-or-large'});continue
|
||||
data=archive.read(item)
|
||||
if b'\0' in data:continue
|
||||
for number,line in enumerate(data.decode(errors='replace').splitlines(),1):
|
||||
for match in REF.finditer(line):
|
||||
refs.append({'path':path,'line':number,'reference':match.group(0),'categories':categories(path)})
|
||||
return {'name':name,'commit':commit,'references':refs,'exclusions':exclusions}
|
||||
|
||||
|
||||
def collect(root):
|
||||
gaps = [
|
||||
'Anonymous forge enumeration cannot attest private repository/package completeness.',
|
||||
'Team membership, deploy keys, webhooks, runners, protected branches and server-side access policies require an authorized owner export.',
|
||||
'Local working trees and public remote-only default-branch archives are scanned; other branches and inaccessible repos are not content-scanned.',
|
||||
'Hidden sibling directories, ignored files, submodule contents outside discovered repos, binary/large/encrypted files and symlinks are not scanned; exclusions are recorded.',
|
||||
'Runtime image/source inventory is collected separately; source scan cannot attest all runtime or external consumers.',
|
||||
'Dynamic/computed org or repository names require owner review; token scan finds literal coulomb/ references only.',
|
||||
]
|
||||
hub = get('http://127.0.0.1:8000/repos/')
|
||||
repos = pages('/api/v1/orgs/coulomb/repos')
|
||||
try:
|
||||
packages = pages('/api/v1/packages/coulomb')
|
||||
except (RuntimeError, OSError):
|
||||
packages = []; gaps.append('Package API unavailable.')
|
||||
paths = {p.resolve() for p in root.iterdir() if p.is_dir() and not p.name.startswith('.') and (p/'.git').exists()}
|
||||
paths.update(Path(r['local_path']).resolve() for r in hub if r.get('local_path') and (Path(r['local_path'])/'.git').exists())
|
||||
paths.discard(SELF)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
||||
scans = list(pool.map(scan, sorted(paths)))
|
||||
server = [{k:r.get(k) for k in ['id','name','full_name','private','archived','default_branch','description']} for r in repos]
|
||||
registered = [{k:r.get(k) for k in ['id','slug','local_path','status','description']} for r in hub]
|
||||
local_names = {m['owner_path'] for s in scans for m in s['remotes'] if m['owner_path']}
|
||||
public_names = {r['full_name'] for r in server}
|
||||
remote_scans=[]
|
||||
for repo in server:
|
||||
if repo['full_name'] not in local_names and repo['name']!=SELF.name:
|
||||
try: remote_scans.append(scan_archive(repo))
|
||||
except (OSError,RuntimeError,ValueError): gaps.append('Remote archive unavailable: '+repo['full_name'])
|
||||
return {'schema_version':1,'forge':{'base_url':BASE,'version':get(BASE+'/api/v1/version'),'authentication':'anonymous'},
|
||||
'g1_complete':False, 'coverage_gaps':gaps,
|
||||
'server_repositories':sorted(server,key=lambda r:r['id']),
|
||||
'registered_repositories':sorted(registered,key=lambda r:r['slug']),
|
||||
'packages': sorted([{'id':p.get('id'),'name':p.get('name'),'type':p.get('type'),'version':p.get('version'),
|
||||
'owner':(p.get('owner') or {}).get('login'),
|
||||
'linked_repository_id':(p.get('repository') or {}).get('id')} for p in packages],key=lambda p:(str(p['id']),str(p['version']))),
|
||||
'reconciliation':{'server_without_local_origin':sorted(public_names-local_names-{f'coulomb/{SELF.name}'}),
|
||||
'local_origin_not_publicly_listed':sorted(local_names-public_names),
|
||||
'coordinator_excluded':str(SELF)},
|
||||
'local_repositories':scans,'remote_archives':remote_scans}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--root',type=Path,default=Path('/home/worsch'))
|
||||
ap.add_argument('--output',type=Path,required=True)
|
||||
ap.add_argument('--compare',type=Path)
|
||||
args=ap.parse_args()
|
||||
result=collect(args.root)
|
||||
encoded=json.dumps(result,sort_keys=True,indent=2)+'\n'
|
||||
args.output.write_text(encoded)
|
||||
print(json.dumps({'repos':len(result['server_repositories']), 'local_repos':len(result['local_repositories']),
|
||||
'references':sum(len(r['references']) for r in result['local_repositories']),
|
||||
'sha256':hashlib.sha256(encoded.encode()).hexdigest(),'g1_complete':False}))
|
||||
if args.compare:
|
||||
same=json.loads(args.compare.read_text())==result
|
||||
print('normalized_equal='+str(same).lower())
|
||||
return 0 if same else 2
|
||||
return 0
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(main())
|
||||
52
tools/runtime_inventory.py
Normal file
52
tools/runtime_inventory.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Project only names, image paths and GitOps source fields; never query Secrets."""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
|
||||
def get(resource):
|
||||
r=subprocess.run(['kubectl','get',resource,'-A','-o','json'],capture_output=True,text=True)
|
||||
if r.returncode:
|
||||
raise RuntimeError('Read failed for '+resource)
|
||||
return json.loads(r.stdout)['items']
|
||||
|
||||
|
||||
def safe_url(value):
|
||||
p=urlsplit(value)
|
||||
return urlunsplit((p.scheme,p.hostname or '',p.path,'','')) if p.scheme else value.split('?')[0]
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--output',required=True)
|
||||
args=ap.parse_args()
|
||||
nodes=get('nodes')
|
||||
addresses={a['address'] for n in nodes for a in n.get('status',{}).get('addresses',[])}
|
||||
if '92.205.62.239' not in addresses:
|
||||
raise RuntimeError('Expected railiance01 node IP not found; refusing wrong-cluster capture')
|
||||
rows=[]
|
||||
for resource in ['deployments','statefulsets','daemonsets','jobs','cronjobs','services.serving.knative.dev','clusters.postgresql.cnpg.io']:
|
||||
for item in get(resource):
|
||||
spec=item.get('spec',{})
|
||||
pod=spec.get('jobTemplate',{}).get('spec',{}).get('template',{}).get('spec',{}) if item['kind']=='CronJob' else spec.get('template',{}).get('spec',{})
|
||||
images=sorted({c['image'] for key in ['containers','initContainers'] for c in pod.get(key,[]) if c.get('image')})
|
||||
if spec.get('imageName'): images.append(spec['imageName'])
|
||||
rows.append({'kind':item['kind'],'namespace':item['metadata'].get('namespace'),'name':item['metadata']['name'],
|
||||
'images':images,'image_pull_secret_names':sorted(x['name'] for x in pod.get('imagePullSecrets',[])),
|
||||
'owners':[{'kind':o['kind'],'name':o['name']} for o in item['metadata'].get('ownerReferences',[])]})
|
||||
apps=[]
|
||||
for i in get('applications.argoproj.io'):
|
||||
s=i['spec']; sources=s.get('sources',[])+([s['source']] if s.get('source') else [])
|
||||
apps.append({'name':i['metadata']['name'],'namespace':i['metadata']['namespace'],'project':s.get('project'),
|
||||
'sources':[{k:safe_url(v) if k=='repoURL' else v for k,v in src.items() if k in ['repoURL','path','chart','targetRevision']} for src in sources]})
|
||||
projects=[{'name':i['metadata']['name'],'namespace':i['metadata']['namespace'],
|
||||
'sourceRepos':[safe_url(x) for x in i['spec'].get('sourceRepos',[])]} for i in get('appprojects.argoproj.io')]
|
||||
result={'cluster':'railiance01','node_ip_verified':'92.205.62.239','coverage':'All namespaces; workload controller units, not pod/replicaset duplicates. Jobs may be historical; owners retained. Secrets never queried.',
|
||||
'workloads':sorted(rows,key=lambda r:(r['namespace'] or '',r['kind'],r['name'])),
|
||||
'applications':sorted(apps,key=lambda r:r['name']),'projects':sorted(projects,key=lambda r:r['name'])}
|
||||
with open(args.output,'w') as f: json.dump(result,f,indent=2,sort_keys=True);f.write('\n')
|
||||
print('Captured',len(rows),'controllers;',len(apps),'Applications;',len(projects),'AppProjects')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
37
tools/verify_inventory.py
Normal file
37
tools/verify_inventory.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fail-closed G6 check of literal old paths against an accepted assignment matrix."""
|
||||
import argparse
|
||||
import json
|
||||
|
||||
|
||||
def check(inventory, matrix):
|
||||
blockers=[]
|
||||
if not inventory.get('g1_complete'):blockers.append('Inventory coverage has not been accepted as complete')
|
||||
if not matrix.get('approved'):blockers.append('Assignment matrix is not approved')
|
||||
assignments={r['repository']:r for r in matrix['assignments']}
|
||||
stale=[]
|
||||
sources=inventory['local_repositories']+inventory.get('remote_archives',[])
|
||||
for repo in sources:
|
||||
for ref in repo['references']:
|
||||
name=ref['reference'].split('/',1)[1].removesuffix('.git')
|
||||
row=assignments.get(name)
|
||||
# Historical matches need explicit disposition, never automatic exemption.
|
||||
if not row or row.get('proposed_org')!='coulomb':
|
||||
stale.append({'repo':repo['name'],**ref})
|
||||
for remote in repo.get('remotes',[]):
|
||||
path=remote.get('owner_path')
|
||||
if path and assignments.get(path.split('/',1)[1],{}).get('proposed_org')!='coulomb':
|
||||
stale.append({'repo':repo['name'],'remote':remote['remote'],'reference':path})
|
||||
if any(not r.get('proposed_org') for r in matrix['assignments']):blockers.append('Unresolved organization assignments')
|
||||
return {'clean':not blockers and not stale,'blockers':blockers,'old_path_findings':len(stale)}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--inventory',required=True);p.add_argument('--assignments',required=True)
|
||||
a=p.parse_args()
|
||||
result=check(json.load(open(a.inventory)),json.load(open(a.assignments)))
|
||||
print(json.dumps(result,indent=2))
|
||||
return 0 if result['clean'] else 1
|
||||
|
||||
if __name__=='__main__':raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue