railiance-apps/tools/forgejo-release-build.py

#!/usr/bin/env python3
"""Build-only public Forgejo owner recipe; no deployment or publication code."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tarfile
import time
import math
import datetime

OWNER=Path(__file__).resolve().parents[1]
INPUTS=OWNER/'build/forgejo'
LOCK=json.loads((INPUTS/'inputs.lock.json').read_text())

def digest(path):
 h=hashlib.sha256()
 with Path(path).open('rb') as f:
  for block in iter(lambda:f.read(1024*1024),b''):h.update(block)
 return h.hexdigest()

def save(path,data):
 Path(path).write_text(json.dumps(data,indent=2,sort_keys=True)+'\n')

def check(path,expected):
 if digest(path)!=expected:raise ValueError('frozen public input digest differs: '+str(path))

def base_env(work):
 return {'PATH':str(work/'node-v22.15.0-linux-x64/bin')+':'+str(work/'go/bin')+':/usr/bin:/bin',
 'HOME':str(work/'home'),'GOTOOLCHAIN':'local','GOPROXY':'off','GOSUMDB':'off','GOVCS':'*:off',
 'GOPATH':str(work/'gopath'),'GOMODCACHE':json.loads((work/'prepared.json').read_text())['module_cache'],
 'GOCACHE':str(work/'gocache'),'CGO_ENABLED':'1','GOMAXPROCS':'4',
 'GOFLAGS':'-trimpath -buildvcs=false -p=4','SOURCE_DATE_EPOCH':str(LOCK['source_date_epoch']),
 'TZ':'UTC','LC_ALL':'C','NODE_OPTIONS':'--max-old-space-size=4096',
 'GIT_CONFIG_NOSYSTEM':'1','GIT_CONFIG_GLOBAL':'/dev/null','GIT_TERMINAL_PROMPT':'0',
 'NPM_CONFIG_USERCONFIG':str(work/'empty.npmrc'),'NPM_CONFIG_GLOBALCONFIG':str(work/'empty-global.npmrc'),
 'NPM_CONFIG_CACHE':str(work/'npm-cache'),'NPM_CONFIG_REGISTRY':'https://registry.npmjs.org',
 'NPM_CONFIG_OFFLINE':'true','NPM_CONFIG_AUDIT':'false','NPM_CONFIG_FUND':'false',
 'NPM_CONFIG_UPDATE_NOTIFIER':'false','PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD':'1'}

def sandbox(work,env):
 cache=Path(env['GOMODCACHE'])
 return ['bwrap','--unshare-net','--unshare-pid','--ro-bind','/usr','/usr','--ro-bind','/lib','/lib',
 '--ro-bind','/lib64','/lib64','--symlink','usr/bin','/bin','--proc','/proc','--dev','/dev',
 '--tmpfs','/tmp','--bind',str(work),str(work),'--ro-bind',str(cache),str(cache),
 '--chdir',str(work/'source'),'--','env','-i']+[k+'='+v for k,v in env.items()]

def run(work,name,args,env=None,offline=False,cwd=None):
 actual=sandbox(work,env)+args if offline else args
 start=time.time()
 with (work/(name+'.log')).open('w') as out:
  proc=subprocess.Popen(actual,cwd=cwd or work,env={"PATH":"/usr/bin:/bin","HOME":str(work/"home")} if offline else env,stdout=out,stderr=subprocess.STDOUT)
  save(work/'process.json',{'phase':name,'pid':proc.pid,'driver_pid':os.getpid(),'argv':args,'state':'running'})
  code=proc.wait()
 result={'phase':name,'argv':args,'returncode':code,'seconds':round(time.time()-start,2)}
 save(work/(name+'-result.json'),result);save(work/'process.json',dict(result,state='complete',pid=None,driver_pid=None))
 print(json.dumps(result),flush=True)
 if code:raise RuntimeError('phase failed; retained log: '+str(work/(name+'.log')))
 return result

def checked_inputs(work):
 for n,h in LOCK['host_tools'].items():check(n,h)
 for n,h in LOCK['lockfiles'].items():check(work/'source'/n,h)
 for x in LOCK['reviewed_outputs']:check(work/'source'/x['path'],x['sha256'])

def prepare(a,work):
 if work.exists():raise ValueError('workspace must be new; existing trial is never restarted or overwritten')
 for n,key in [('source_archive','source_archive_sha256'),('go_archive','go_archive_sha256'),('node_archive','node_archive_sha256')]:
  if not getattr(a,n):raise ValueError('explicit '+n+' required')
  check(getattr(a,n),LOCK[key])
 if not a.module_cache:raise ValueError('explicit public module cache required')
 cache=Path(a.module_cache).resolve();assert cache.is_dir()
 check(INPUTS/LOCK['patch_file'],LOCK['patch_sha256'])
 for n,h in LOCK['host_tools'].items():check(n,h)
 work.mkdir(parents=True);(work/'home').mkdir()
 with tarfile.open(a.source_archive) as t:
  # The official archive is digest-bound before any member is extracted.
  parent=work/'unpacked';parent.mkdir();t.extractall(parent,filter='data')
  folders=list(parent.iterdir());assert len(folders)==1 and folders[0].is_dir();folders[0].rename(work/'source');parent.rmdir()
 for n in ['go_archive','node_archive']:
  with tarfile.open(getattr(a,n)) as t:t.extractall(work,filter='data')
 for n in ['empty.npmrc','empty-global.npmrc']:(work/n).write_text('')
 if a.go_cache_seed:shutil.copytree(Path(a.go_cache_seed).resolve(),work/'gocache')
 if a.npm_cache_seed:shutil.copytree(Path(a.npm_cache_seed).resolve(),work/'npm-cache')
 subprocess.run(['/usr/bin/git','apply','--check',str(INPUTS/LOCK['patch_file'])],cwd=work/'source',check=True,env={'PATH':'/usr/bin:/bin','HOME':str(work/'home'),'GIT_CONFIG_NOSYSTEM':'1','GIT_CONFIG_GLOBAL':'/dev/null'})
 subprocess.run(['/usr/bin/git','apply',str(INPUTS/LOCK['patch_file'])],cwd=work/'source',check=True,env={'PATH':'/usr/bin:/bin','HOME':str(work/'home'),'GIT_CONFIG_NOSYSTEM':'1','GIT_CONFIG_GLOBAL':'/dev/null'})
 save(work/'prepared.json',{'module_cache':str(cache),'lock_sha256':digest(INPUTS/'inputs.lock.json'),'public_archives_verified':True})
 checked_inputs(work)
 packages=json.loads((work/'source/package-lock.json').read_text())['packages']
 for name,package in packages.items():
  if not name:continue
  if not package.get('resolved','').startswith('https://registry.npmjs.org/') or not package.get('integrity','').startswith('sha512-'):
   raise ValueError('non-public/non-integrity-bound npm package: '+name)
 print(json.dumps({'prepared':True,'lockfile_npm_packages':len(packages)-1}))

def normalize(folder):
 epoch=LOCK['source_date_epoch']
 for root,dirs,files in os.walk(folder):
  dirs[:]=[x for x in dirs if x not in ('node_modules','.git')]
  for n in files:
   p=Path(root)/n
   if not p.is_symlink():os.utime(p,(epoch,epoch))
  os.utime(root,(epoch,epoch))

def build(work):
 checked_inputs(work);env=base_env(work)
 run(work,'verify-go-modules',['go','mod','verify'],env,True)
 run(work,'npm-offline-ci',['npm','ci','--offline','--ignore-scripts','--no-audit','--no-fund'],env,True)
 normalize(work/'source')
 version=['GOFLAGS=-trimpath -buildvcs=false -p=4','RELEASE_VERSION='+LOCK['release_version'],'FORGEJO_VERSION='+LOCK['release_version'],'FORGEJO_VERSION_API='+LOCK['release_version']]
 run(work,'frontend',['make']+version+['frontend'],env,True)
 for n in ['public','options','templates','modules/migration/schemas']:normalize(work/'source'/n)
 run(work,'embedded-and-static-backend',['make']+version+['TAGS='+LOCK['tags'],'go-check','generate-backend','static-executable'],env,True)
 checked_inputs(work)
 save(work/'build.json',{'binary_sha256':digest(work/'source/gitea'),'source_locks_preserved':True,'frontend_built':True,'tags':LOCK['tags'],'image_deployment_selected':False})

def verify(work):
 checked_inputs(work);binary=work/'source/gitea'
 target=work/'embedded-extracted'
 if target.exists():raise ValueError('retain prior verification; extraction target already exists')
 target.mkdir()
 # No source, asset tree, host /usr, libraries, configuration or HOME is
 # exposed to the executable. A filesystem fallback cannot pass this proof.
 bare=['bwrap','--unshare-net','--unshare-pid','--proc','/proc','--dev','/dev',
       '--tmpfs','/tmp','--dir','/app','--ro-bind',str(binary),'/app/forgejo',
       '--bind',str(target),'/results','--chdir','/tmp','--clearenv']
 for k,v in {'PATH':'/nonexistent','HOME':'/tmp/home','USER':'builder','USERNAME':'builder',
             'GITEA_WORK_DIR':'/tmp/work','GITEA_CUSTOM':'/tmp/custom',
             'GITEA_APP_INI':'/tmp/absent-app.ini','TZ':'UTC','LC_ALL':'C'}.items():
  bare+=['--setenv',k,v]
 bare+=['--','/app/forgejo']
 outer={'PATH':'/usr/bin:/bin','HOME':str(work/'home')}
 run(work,'binary-version',bare+['--version'],outer)
 run(work,'new-cli-help',bare+['admin','auth','add-oauth','--help'],outer)
 assert '--required-claim-values' in (work/'new-cli-help.log').read_text()
 assert '--secret-file' in (work/'new-cli-help.log').read_text()
 run(work,'embedded-extract',bare+['embedded','extract','--include-vendored','--destination','/results','**'],outer)
 records={}
 for folder in ['public','options','templates']:
  for f in sorted((work/'source'/folder).rglob('*')):
   if not f.is_file():continue
   n=str(f.relative_to(work/'source'));q=target/n
   if not q.is_file() or digest(f)!=digest(q):raise ValueError('embedded asset missing/different: '+n)
   records[n]=digest(f)
 assert 'public/assets/js/index.js' in records and 'public/assets/css/index.css' in records
 assert 'public/assets/licenses.txt' in records
 save(work/'embedded-files.json',records)
 readelf=subprocess.run(['/usr/bin/readelf','-l',str(binary)],capture_output=True,text=True,check=True,env={'PATH':'/usr/bin:/bin'})
 assert 'INTERP' not in readelf.stdout
 save(work/'verification.json',{'binary_sha256':digest(binary),'embedded_files':len(records),
  'embedded_file_manifest_sha256':digest(work/'embedded-files.json'),'all_public_options_templates_exact':True,
  'no_source_assets_libraries_or_host_config_mounted':True,
  'no_dynamic_interpreter':True,'new_cli_flag_present':True,'network':'bwrap --unshare-net',
  'source_locks_preserved':True,'architecture_approval':False,'native_admission':False})
 print((work/'verification.json').read_text())

def container(work):
 verified=json.loads((work/'verification.json').read_text());check(work/'source/gitea',verified['binary_sha256'])
 context=work/'container-context'
 if context.exists():raise ValueError('container trial already exists; retain evidence before an explicit revised trial')
 archive_path=work/'image.oci.tar'
 if archive_path.exists():raise ValueError('retain existing OCI artifact; never overwrite it')
 base_bytes=(INPUTS/'runtime-base-config.json').read_bytes()
 assert hashlib.sha256(base_bytes).hexdigest()==LOCK['runtime_base_config_sha256']
 original=json.loads(base_bytes)
 epoch=max(LOCK['source_date_epoch'],math.ceil(datetime.datetime.fromisoformat(original['created'].replace('Z','+00:00')).timestamp()))
 context.mkdir();shutil.copyfile(work/'source/gitea',context/'gitea')
 os.utime(context/'gitea',(LOCK['source_date_epoch'],)*2)
 (context/'Dockerfile').write_text('FROM '+LOCK['runtime_base']+'\nARG SOURCE_DATE_EPOCH\nLABEL org.opencontainers.image.version="'+LOCK['release_version']+'" org.opencontainers.image.revision="'+LOCK['source_commit']+'" org.railiance.source.patch.sha256="'+LOCK['patch_sha256']+'"\nCOPY --chown=0:0 --chmod=755 gitea /app/gitea/gitea\n')
 os.utime(context/'Dockerfile',(LOCK['source_date_epoch'],)*2)
 config=work/'docker-config';config.mkdir(exist_ok=True)
 env={'PATH':'/usr/bin:/bin','HOME':str(work/'home'),'DOCKER_CONFIG':str(config),'DOCKER_HOST':'unix:///var/run/docker.sock'}
 # SOURCE_DATE_EPOCH alone does not normalise COPY parent-directory mtimes.
 # OCI output permits layer rewriting without the Docker-store unpack conflict.
 # The epoch never predates the immutable base, preserving its exact layer prefix.
 run(work,'container-build',['docker','buildx','build','--output','type=oci,dest='+str(archive_path)+',rewrite-timestamp=true',
  '--no-cache','--platform',LOCK['platform'],'--pull=false','--network=none','--provenance=false','--sbom=false',
  '--metadata-file',str(work/'image-build-metadata.json'),'--build-arg','SOURCE_DATE_EPOCH='+str(epoch),str(context)],env)
 metadata=json.loads((work/'image-build-metadata.json').read_text())
 with tarfile.open(archive_path) as archive:
  index_bytes=archive.extractfile('index.json').read();index=json.loads(index_bytes)
  assert len(index['manifests'])==1
  descriptor=index['manifests'][0]
  assert descriptor['platform']=={'architecture':'amd64','os':'linux'}
  assert descriptor['digest']==metadata['containerimage.digest']
  def blob(d):
   value=archive.extractfile('blobs/sha256/'+d.split(':')[1]).read()
   assert 'sha256:'+hashlib.sha256(value).hexdigest()==d
   return value
  manifest=json.loads(blob(descriptor['digest']))
  config_digest=manifest['config']['digest'];config=json.loads(blob(config_digest))
  assert config_digest==metadata['containerimage.config.digest']
  for layer in manifest['layers']:blob(layer['digest'])
  assert config['rootfs']['diff_ids'][:-1]==original['rootfs']['diff_ids']
  assert config['architecture']=='amd64' and config['os']=='linux'
  for key in ['User','Entrypoint','Cmd','Env','WorkingDir','Volumes']:
   assert config['config'].get(key)==original['config'].get(key),key
  labels=dict(original['config'].get('Labels',{}))
  labels.update({'org.opencontainers.image.version':LOCK['release_version'],
   'org.opencontainers.image.revision':LOCK['source_commit'],'org.railiance.source.patch.sha256':LOCK['patch_sha256']})
  assert config['config'].get('Labels')==labels
  assert config['config']['User']=='1000:1000'
  expected_created=datetime.datetime.fromtimestamp(epoch,datetime.timezone.utc).isoformat(timespec='seconds').replace('+00:00','Z')
  assert config['created']==expected_created
  found=False
  last='blobs/sha256/'+manifest['layers'][-1]['digest'].split(':')[1]
  with tarfile.open(fileobj=archive.extractfile(last),mode='r|*') as layer:
   for member in layer:
    name=member.name.lstrip('./');assert name in ('app','app/gitea','app/gitea/gitea')
    if name=='app/gitea/gitea':
     h=hashlib.sha256();stream=layer.extractfile(member)
     for block in iter(lambda:stream.read(1024*1024),b''):h.update(block)
     assert h.hexdigest()==verified['binary_sha256'];found=True
  assert found
 save(work/'container-result.json',{'image_manifest_digest':descriptor['digest'],'config_digest':config_digest,
  'oci_index_sha256':hashlib.sha256(index_bytes).hexdigest(),'archive_sha256':digest(archive_path),
  'artifact_path':str(archive_path),'source_epoch':LOCK['source_date_epoch'],'container_epoch':epoch,
  'all_base_layers_and_rootless_runtime_config_preserved':True,'all_oci_blob_digests_verified':True,
  'final_layer_binary_exact':True,'registry_publication':False,'deployment_tag':None,
  'host_or_secret_mounts':False,'privileged_containers':False,'local_runtime_image_loaded':False})
 print((work/'container-result.json').read_text())

def main():
 parser=argparse.ArgumentParser(description=__doc__)
 parser.add_argument('phase',choices=['prepare','acquire-npm','build','verify','container'])
 parser.add_argument('--workspace',required=True)
 for n in ['source-archive','go-archive','node-archive','module-cache','go-cache-seed','npm-cache-seed']:parser.add_argument('--'+n)
 a=parser.parse_args();work=Path(a.workspace).resolve()
 if not str(work).startswith('/tmp/'):raise ValueError('isolated build workspace must be under /tmp')
 if a.phase=='prepare':return prepare(a,work)
 assert json.loads((work/'prepared.json').read_text())['lock_sha256']==digest(INPUTS/'inputs.lock.json')
 if a.phase=='acquire-npm':
  checked_inputs(work);env=base_env(work);env['NPM_CONFIG_OFFLINE']='false'
  return run(work,'npm-public-acquire',['npm','ci','--ignore-scripts','--no-audit','--no-fund'],env,cwd=work/'source')
 {'build':build,'verify':verify,'container':container}[a.phase](work)

if __name__=='__main__':main()