From 83c521bf2fa351d730a90bf7f7e13dfd4b0cff2a Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 17 Oct 2026 00:24:44 +0200 Subject: [PATCH] perf(security): batch small staged blobs with bounded volatile storage Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1468b-63fa-7710-85dc-8f2e16b3ebfa --- src/repo_manager/secret_scan.py | 340 ++++++++++++++++-- tests/test_secret_scan.py | 262 +++++++++++++- .../RMGR-WP-0021-secret-detection-controls.md | 43 ++- 3 files changed, 615 insertions(+), 30 deletions(-) diff --git a/src/repo_manager/secret_scan.py b/src/repo_manager/secret_scan.py index 9422eb5..c2203bd 100644 --- a/src/repo_manager/secret_scan.py +++ b/src/repo_manager/secret_scan.py @@ -7,20 +7,42 @@ claim history/push/CI coverage. Its caller must treat unverified as refusal. from __future__ import annotations import argparse +import contextlib +import ctypes +import fcntl +import functools import hashlib import json import os import re +import shutil +import signal +import resource import stat +import sys import subprocess import tempfile +import time from pathlib import Path +# Resolve libc and prctl before fork; child_limits performs no library loading. +_LIBC = ctypes.CDLL(None, use_errno=True) +_PRCTL = _LIBC.prctl +_PRCTL.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] +_PRCTL.restype = ctypes.c_int + VERSION = '8.30.1' BINARY_SHA256 = '88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509' MAX_BLOB_BYTES = 8 * 1024 * 1024 MAX_TOTAL_BYTES = 64 * 1024 * 1024 MAX_BLOBS = 500 +MAX_BATCH_BLOBS = 64 +MAX_BATCH_BYTES = 8 * 1024 * 1024 +# PIPE_BUF bytes fit the original stdin writer's single atomic write. Larger +# blobs retain the stdin path, avoiding a change to fragment boundaries. +DIRECTORY_BLOB_BYTES = 4096 +MAX_REPORT_BYTES = 1024 * 1024 +SCAN_SECONDS = 120 CONFIG = '[extend]\nuseDefault = true\n' @@ -31,7 +53,8 @@ class Unverified(Exception): def git(repo: Path, *args: str) -> bytes: env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'} result = subprocess.run(['git', '-C', str(repo), *args], stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, env=env, timeout=20, check=False) + stderr=subprocess.DEVNULL, env=env, timeout=20, check=False, + preexec_fn=functools.partial(child_limits, MAX_TOTAL_BYTES, os.getpid())) if result.returncode: raise Unverified('git-read-failed') return result.stdout @@ -63,10 +86,167 @@ def staged_objects(repo: Path) -> tuple[bytes, list[str]]: return snapshot, sorted(objects) +# Source bytes must never be copied onto persistent storage. The watchdog owns +# no source data; its pipe EOF detects parent death without a service or hook. +_CLEANUP = """ +import os, select, shutil, signal, stat, sys +path, dev, ino, uid, parent = sys.argv[1:] +root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) +if os.getppid() != int(parent): + sys.exit(4) +pid = os.pidfd_open(int(parent)) +if os.getppid() != int(parent): + sys.exit(4) +os.write(1, b'R') +ready, _, _ = select.select([0, pid], [], [], 180) +# Exceptional abort targets only the exact parent, never a process group. +# If kernel quiescence stalls, retain this observer/duty until death is proven. +if not ready: + try: + signal.pidfd_send_signal(pid, signal.SIGTERM) + except ProcessLookupError: + pass + dead, _, _ = select.select([pid], [], [], 5) + if not dead: + try: + signal.pidfd_send_signal(pid, signal.SIGKILL) + except ProcessLookupError: + pass + select.select([pid], [], []) +name = os.path.basename(path) +try: + info = os.stat(name, dir_fd=root, follow_symlinks=False) +except FileNotFoundError: + sys.exit(0) +if (info.st_dev, info.st_ino, info.st_uid) != (int(dev), int(ino), int(uid)): + sys.exit(3) +if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700: + sys.exit(3) +shutil.rmtree(name, dir_fd=root) +""" + + +def is_tmpfs(directory: Path) -> bool: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + result = ctypes.create_string_buffer(256) + if _LIBC.fstatfs(descriptor, ctypes.byref(result)): + raise OSError(ctypes.get_errno(), 'statfs failed') + return ctypes.c_long.from_buffer(result).value == 0x01021994 + finally: + os.close(descriptor) + + +def remove_storage(directory: Path, identity: os.stat_result) -> None: + root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + current = os.stat(directory.name, dir_fd=root, follow_symlinks=False) + except FileNotFoundError: + return + if ((current.st_dev, current.st_ino, current.st_uid) != + (identity.st_dev, identity.st_ino, identity.st_uid) + or not stat.S_ISDIR(current.st_mode) + or stat.S_IMODE(current.st_mode) != 0o700): + raise Unverified('volatile-storage-identity-changed') + shutil.rmtree(directory.name, dir_fd=root) + finally: + os.close(root) + + +@contextlib.contextmanager +def private_storage(): + if not is_tmpfs(Path('/dev/shm')): + raise Unverified('volatile-storage-unavailable') + temporary = tempfile.mkdtemp(prefix='rmgr-secret-scan-', dir='/dev/shm') + directory = Path(temporary) + identity = directory.lstat() + watcher = None + try: + if not is_tmpfs(directory): + raise Unverified('volatile-storage-unavailable') + watcher = subprocess.Popen([sys.executable, '-I', '-c', _CLEANUP, temporary, + str(identity.st_dev), str(identity.st_ino), str(identity.st_uid), str(os.getpid())], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env={'PATH': '/usr/bin:/bin', 'LANG': 'C'}, close_fds=True) + import select + ready, _, _ = select.select([watcher.stdout], [], [], 5) + if not ready or watcher.stdout.read(1) != b'R' or watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + watcher.stdout.close() + yield directory, watcher + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + finally: + pending_error = sys.exc_info()[0] is not None + # All synchronous Git/scanner children have exited before yield unwinds. + try: + remove_storage(directory, identity) + finally: + if watcher is not None: + watcher.stdin.close() + watcher.stdout.close() + try: + code = watcher.wait(timeout=5) + except subprocess.TimeoutExpired: + watcher.kill() + code = watcher.wait(timeout=5) + if code and not pending_error: + raise Unverified('cleanup-watchdog-failed') + + +def child_limits(file_bytes: int, parent_pid: int) -> None: + # Parent loss must quiesce every source-reading/writing child. Install the + # signal before any exec and close the fork/prctl race using the exact PID. + if _PRCTL(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent_pid: + os._exit(127) + # Linux adapter already requires /proc/self/fd. Limits apply only to children. + resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes)) + resource.setrlimit(resource.RLIMIT_CPU, (20, 20)) + resource.setrlimit(resource.RLIMIT_AS, (8 * 1024**3, 8 * 1024**3)) + + +def object_batch(repo: Path, objects: list[str], directory: Path) -> list[tuple[str, bytes]]: + """Bound Git's output before capture, then validate the complete framing.""" + request = ''.join(oid + '\n' for oid in objects).encode('ascii') + env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'} + with tempfile.TemporaryFile(dir=directory) as output: + result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch'], + input=request, stdout=output, stderr=subprocess.DEVNULL, + env=env, timeout=20, check=False, + preexec_fn=functools.partial(child_limits, + MAX_BATCH_BYTES + len(objects) * 128, os.getpid())) + if result.returncode: + raise Unverified('git-read-failed') + output.seek(0) + blobs = [] + for oid in objects: + header = output.readline(128) + match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)\n', header) + if not match or match[1].decode() != oid: + raise Unverified('invalid-object-batch') + size = int(match[2]) + if size > MAX_BLOB_BYTES or size > MAX_BATCH_BYTES: + raise Unverified('staged-byte-limit') + content = output.read(size) + if len(content) != size or output.read(1) != b'\n': + raise Unverified('blob-size-mismatch') + algorithm = 'sha1' if len(oid) == 40 else 'sha256' + digest = hashlib.new(algorithm, b'blob ' + str(size).encode() + b'\0' + content) + if digest.hexdigest() != oid: + raise Unverified('blob-digest-mismatch') + blobs.append((oid, content)) + if output.read(1): + raise Unverified('invalid-object-batch') + return blobs + + def scan_staged(repo: Path, binary: Path) -> dict: """Return only fixed statuses/counts and Git blob IDs, never finding contents.""" scanned = 0 descriptor = None + executable = None + deadline = time.monotonic() + SCAN_SECONDS try: descriptor = os.open(binary, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) info = os.fstat(descriptor) @@ -74,47 +254,149 @@ def scan_staged(repo: Path, binary: Path) -> dict: or not info.st_mode & 0o111 or info.st_uid not in (0, os.getuid()) or info.st_size > 64 * 1024 * 1024): raise Unverified('scanner-file-refused') + executable = os.memfd_create('rmgr-verified-scanner', os.MFD_ALLOW_SEALING) digest = hashlib.sha256() + copied = 0 while chunk := os.read(descriptor, 1024 * 1024): + copied += len(chunk) + if copied > 64 * 1024 * 1024: + raise Unverified('scanner-file-refused') digest.update(chunk) + with memoryview(chunk) as view: + while view: + written = os.write(executable, view) + if written <= 0: + raise Unverified('scanner-file-refused') + view = view[written:] if digest.hexdigest() != BINARY_SHA256: raise Unverified('scanner-digest-mismatch') + fcntl.fcntl(executable, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW + | fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL) snapshot, objects = staged_objects(repo) blocked = [] - total = 0 - # Do not honor repository-local config/ignore files, comments or ambient - # GITLEAKS settings. No scanner report or credential-bearing diagnostic is kept. - with tempfile.TemporaryDirectory(prefix='rmgr-secret-scan-') as temporary: - directory = Path(temporary) + # Neither source names nor repository config/ignore files enter this + # private directory. Reports contain ONLY neutral blob paths. + with private_storage() as (directory, watcher): + temporary = str(directory) config = directory / 'scanner.toml'; config.write_text(CONFIG) ignore = directory / '.gitleaksignore'; ignore.write_text('') - env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C', 'TMPDIR': temporary} - for oid in objects: - size = int(git(repo, 'cat-file', '-s', oid)) - total += size - if size < 0 or size > MAX_BLOB_BYTES or total > MAX_TOTAL_BYTES: - raise Unverified('staged-byte-limit') - content = git(repo, 'cat-file', 'blob', oid) - if len(content) != size: + template = directory / 'report.tmpl' + template.write_text('{{range .}}{{.File}}\n{{end}}') + env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C', + 'TMPDIR': temporary, 'GOMAXPROCS': '2'} + # One bounded metadata query replaces a size subprocess per object. + request = ''.join(oid + '\n' for oid in objects).encode('ascii') + with tempfile.TemporaryFile(dir=directory) as output: + result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch-check'], + input=request, stdout=output, stderr=subprocess.DEVNULL, + env={**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}, timeout=20, + check=False, preexec_fn=functools.partial(child_limits, MAX_BLOBS * 128, os.getpid())) + if result.returncode: + raise Unverified('git-read-failed') + output.seek(0) + metadata = output.read(MAX_BLOBS * 128 + 1).splitlines() + if len(metadata) != len(objects): + raise Unverified('invalid-object-batch') + sizes = [] + for oid, line in zip(objects, metadata): + match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)', line) + if not match or match[1].decode() != oid: + raise Unverified('invalid-object-batch') + sizes.append(int(match[2])) + if any(size > MAX_BLOB_BYTES for size in sizes) or sum(sizes) > MAX_TOTAL_BYTES: + raise Unverified('staged-byte-limit') + + def invoke(mode: str, content: bytes | None = None) -> list[str]: + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + remaining = deadline - time.monotonic() + if remaining <= 0: + raise Unverified('scan-deadline-exceeded') + report = directory / 'report.paths' + report.unlink(missing_ok=True) + command = [str(binary), mode, '--config', str(config), + '--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow', + '--redact=100', '--no-banner', '--no-color', '--log-level', 'warn', + '--exit-code', '20', '--timeout', '15'] + if mode == 'dir': + command += ['--report-format', 'template', '--report-template', str(template), '--report-path', str(report), + '--max-target-megabytes', '0', 'blobs'] + with tempfile.TemporaryFile(dir=directory) as diagnostics: + result = subprocess.run(command, executable=f'/proc/self/fd/{executable}', + pass_fds=(executable,), input=content, cwd=directory, env=env, + stdout=subprocess.DEVNULL, stderr=diagnostics, + timeout=min(20, remaining), check=False, + preexec_fn=functools.partial(child_limits, MAX_REPORT_BYTES, os.getpid())) + # The scanner can warn and skip a file without an error exit. + # Refuse ALL warnings/errors, including partial traversal. + diagnostics.seek(0) + messages = diagnostics.read(MAX_REPORT_BYTES + 1) + # v8.30.1 emits exactly this summary for a successful blocked + # scan. Every other warning/error is a refusal, never output. + summary = re.fullmatch( + rb'[0-9]{1,2}:[0-9]{2}(?:AM|PM) WRN leaks found: [1-9][0-9]*\n', + messages) + if (result.returncode not in (0, 20) + or (messages and not (result.returncode == 20 and summary))): + raise Unverified('scanner-failed') + if mode == 'stdin': + return ['stdin'] if result.returncode == 20 else [] + if not report.exists() or report.stat().st_size > MAX_REPORT_BYTES: + raise Unverified('scanner-report-refused') + paths = report.read_bytes().splitlines() + known = {('blobs/' + oid).encode(): oid for oid in small} + if any(path not in known for path in paths): + raise Unverified('scanner-report-refused') + findings = sorted({known[path] for path in paths}) + if bool(findings) != (result.returncode == 20): + raise Unverified('scanner-report-refused') + return findings + + offset = 0 + while offset < len(objects): + batch = [] + batch_bytes = 0 + while offset < len(objects) and len(batch) < MAX_BATCH_BLOBS: + size = sizes[offset] + if batch and batch_bytes + size > MAX_BATCH_BYTES: + break + batch.append(objects[offset]); batch_bytes += size; offset += 1 + if watcher.poll() is not None: + raise Unverified('cleanup-watchdog-unavailable') + blobs = object_batch(repo, batch, directory) + # Check metadata again against exact bytes, including same-size + # substitution via corrupt object storage. Hashes are checked above. + expected = dict(zip(objects, sizes)) + if any(len(content) != expected[oid] for oid, content in blobs): raise Unverified('blob-size-mismatch') - command = [str(binary), 'stdin', '--config', str(config), - '--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow', - '--redact=100', '--no-banner', '--no-color', '--log-level', 'fatal', - '--exit-code', '20', '--timeout', '15'] - result = subprocess.run(command, executable=f'/proc/self/fd/{descriptor}', - pass_fds=(descriptor,), input=content, cwd=directory, env=env, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=20, check=False) - if result.returncode not in (0, 20): - raise Unverified('scanner-failed') - scanned += 1 - if result.returncode == 20: - blocked.append(oid) + payload = directory / 'blobs'; payload.mkdir(mode=0o700) + small = [] + try: + for oid, content in blobs: + if len(content) > DIRECTORY_BLOB_BYTES: + if invoke('stdin', content): + blocked.append(oid) + scanned += 1 + else: + path = payload / oid + with path.open('xb') as target: + target.write(content) + path.chmod(0o600) + small.append(oid) + if small: + blocked.extend(invoke('dir')) + scanned += len(small) + finally: + for path in payload.iterdir(): + path.unlink() + payload.rmdir() if git(repo, 'ls-files', '--stage', '-z') != snapshot: raise Unverified('index-changed-during-scan') + if time.monotonic() > deadline: + raise Unverified('scan-deadline-exceeded') return {'status': 'blocked' if blocked else 'clean', 'scope': 'changed-staged-blobs', 'scanner_version': VERSION, 'scanner_sha256': BINARY_SHA256, - 'scanned_blobs': scanned, 'blocked_blob_ids': blocked} + 'scanned_blobs': scanned, 'blocked_blob_ids': sorted(blocked)} except Unverified as error: reason = error.args[0] except (OSError, ValueError, subprocess.SubprocessError): @@ -122,6 +404,8 @@ def scan_staged(repo: Path, binary: Path) -> dict: finally: if descriptor is not None: os.close(descriptor) + if executable is not None: + os.close(executable) return {'status': 'unverified', 'scope': 'changed-staged-blobs', 'reason': reason, 'scanned_blobs': scanned} diff --git a/tests/test_secret_scan.py b/tests/test_secret_scan.py index a8768aa..d3555f9 100644 --- a/tests/test_secret_scan.py +++ b/tests/test_secret_scan.py @@ -182,7 +182,7 @@ def test_scanner_failure_never_passes_or_emits_diagnostics(binary, repo, monkeyp stage(repo, b'clean\n') original = secret_scan.subprocess.run def broken(command, **kwargs): - if len(command) > 1 and command[1] == 'stdin': + if len(command) > 1 and command[1] in ('stdin', 'dir'): if failure == 'timeout': raise subprocess.TimeoutExpired(command, 20, output=b'synthetic-sensitive-diagnostic') return subprocess.CompletedProcess(command, 2) @@ -210,3 +210,263 @@ def test_actual_commit_only_hook_scans_its_temporary_index(binary, repo, private assert git(repo, 'rev-parse', 'HEAD') == before assert bool(private_key.strip() in result.stdout + result.stderr) is False assert b'"status": "blocked"' in result.stdout + result.stderr + + +def test_empty_and_deletion_only_index(binary, repo): + assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0 + stage(repo, b'clean\n'); git(repo, 'commit', '-qm', 'seed') + assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0 + git(repo, 'rm', '-q', 'application.txt') + assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0 + + +def test_duplicate_blobs_and_exact_attribution(binary, repo, private_key): + stage(repo, private_key, 'secret.png') # original filename cannot allowlist content + stage(repo, private_key, 'duplicate.txt') + stage(repo, b'clean\n', 'clean.txt') + oid = git(repo, 'rev-parse', ':secret.png').decode().strip() + result = secret_scan.scan_staged(repo, binary) + assert result['status'] == 'blocked' + assert result['scanned_blobs'] == 2 + assert result['blocked_blob_ids'] == [oid] + + +@pytest.mark.parametrize('size', [4096, 4097, 100000, 100001]) +def test_small_batch_and_large_fallback_equivalence(binary, repo, private_key, size): + data = b'\n' * (size - len(private_key)) + private_key + stage(repo, data) + result = secret_scan.scan_staged(repo, binary) + assert result['status'] == 'blocked' + assert result['blocked_blob_ids'] == [git(repo, 'rev-parse', ':application.txt').decode().strip()] + + +def test_file_boundaries_never_synthesize_a_finding(binary, repo, private_key): + cut = private_key.index(b'\n') // 2 + stage(repo, private_key[:cut], 'first') + stage(repo, private_key[cut:], 'second') + # Each independent fragment matches the original single-blob stdin outcome; + # concatenation would synthesize the complete private-key opening marker. + result = secret_scan.scan_staged(repo, binary) + assert result['status'] == 'clean' + assert result['scanned_blobs'] == 2 + + +def test_binary_and_encoded_fixture_equivalence(binary, repo, private_key, monkeypatch): + import base64 + original = secret_scan.DIRECTORY_BLOB_BYTES + for data in [b'\x00\x01\x02\n' + private_key, base64.b64encode(private_key)]: + stage(repo, data) + monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', 0) + stdin = secret_scan.scan_staged(repo, binary) + monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', original) + batched = secret_scan.scan_staged(repo, binary) + assert batched == stdin + assert batched['status'] == 'blocked' + + +def test_batch_count_bound_and_independent_findings(binary, repo, private_key, monkeypatch): + monkeypatch.setattr(secret_scan, 'MAX_BATCH_BLOBS', 3) + for i in range(7): + stage(repo, (f'fixture {i}\n'.encode() + private_key) if i % 2 else f'clean {i}\n'.encode(), str(i)) + result = secret_scan.scan_staged(repo, binary) + expected = sorted(git(repo, 'rev-parse', ':' + str(i)).decode().strip() for i in [1, 3, 5]) + assert result['status'] == 'blocked' + assert result['scanned_blobs'] == 7 + assert result['blocked_blob_ids'] == expected + + +def test_total_and_object_count_bounds_refuse(binary, repo, monkeypatch): + stage(repo, b'12345', 'a'); stage(repo, b'67890', 'b') + monkeypatch.setattr(secret_scan, 'MAX_TOTAL_BYTES', 9) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-byte-limit' + monkeypatch.setattr(secret_scan, 'MAX_BLOBS', 1) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-blob-limit' + + +@pytest.mark.parametrize('mutation', ['path', 'in-place']) +def test_verified_executable_snapshot_survives_mutation(binary, repo, monkeypatch, tmp_path, mutation): + import shutil + scanner = tmp_path / 'scanner'; shutil.copyfile(binary, scanner); scanner.chmod(0o700) + stage(repo, b'clean\n') + original = secret_scan.staged_objects + def mutate(repo): + if mutation == 'path': + scanner.unlink() + scanner.write_bytes(b'not executable scanner bytes\n') + scanner.chmod(0o700) + return original(repo) + monkeypatch.setattr(secret_scan, 'staged_objects', mutate) + assert secret_scan.scan_staged(repo, scanner)['status'] == 'clean' + + +@pytest.mark.parametrize('fault', ['warning', 'unknown-path', 'missing-report', 'returncode']) +def test_partial_scans_and_invalid_reports_refuse(binary, repo, monkeypatch, fault): + stage(repo, b'clean\n') + original = secret_scan.subprocess.run + def fail(command, **kwargs): + result = original(command, **kwargs) + if command[1] == 'dir': + report = Path(command[command.index('--report-path') + 1]) + if fault == 'warning': + kwargs['stderr'].write(b'synthetic-sensitive-diagnostic\n') + elif fault == 'unknown-path': + report.write_text('blobs/unknown\n') + elif fault == 'missing-report': + report.unlink() + else: + return subprocess.CompletedProcess(command, 20) + return result + monkeypatch.setattr(secret_scan.subprocess, 'run', fail) + result = secret_scan.scan_staged(repo, binary) + assert result['status'] == 'unverified' + assert 'synthetic-sensitive-diagnostic' not in json.dumps(result) + + +def test_deadline_exhaustion_refuses(binary, repo, monkeypatch): + stage(repo, b'clean\n') + monkeypatch.setattr(secret_scan, 'SCAN_SECONDS', 0) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'scan-deadline-exceeded' + + +def test_git_object_output_corruption_refuses(binary, repo, monkeypatch): + stage(repo, b'clean\n') + original = secret_scan.subprocess.run + def corrupt(command, **kwargs): + result = original(command, **kwargs) + if command[-1] == '--batch': + output = kwargs['stdout'] + output.seek(-3, 2); output.write(b'xx\n') + return result + monkeypatch.setattr(secret_scan.subprocess, 'run', corrupt) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'blob-digest-mismatch' + + +def test_report_growth_limit_refuses(binary, repo, private_key, monkeypatch): + stage(repo, private_key) + monkeypatch.setattr(secret_scan, 'MAX_REPORT_BYTES', 16) + assert secret_scan.scan_staged(repo, binary)['status'] == 'unverified' + + +def test_sealed_executable_rejects_write(binary, repo, monkeypatch): + import errno + stage(repo, b'clean\n') + original = secret_scan.subprocess.run + checked = [] + def assert_sealed(command, **kwargs): + if command[1] in ('dir', 'stdin'): + fd = kwargs['pass_fds'][0] + with pytest.raises(OSError) as error: + os.pwrite(fd, b'X', 0) + assert error.value.errno == errno.EPERM + checked.append(True) + return original(command, **kwargs) + monkeypatch.setattr(secret_scan.subprocess, 'run', assert_sealed) + assert secret_scan.scan_staged(repo, binary)['status'] == 'clean' + assert checked + + +def test_nonvolatile_storage_refuses(binary, repo, monkeypatch): + stage(repo, b'clean\n') + monkeypatch.setattr(secret_scan, 'is_tmpfs', lambda directory: False) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'volatile-storage-unavailable' + + +def test_cleanup_watchdog_loss_refuses(binary, repo, monkeypatch): + stage(repo, b'clean\n') + original = secret_scan.object_batch + def kill_watchdog(repo, objects, directory): + # Discover only our child watchdog using the context manager wrapper. + watcher.kill(); watcher.wait(timeout=5) + return original(repo, objects, directory) + original_storage = secret_scan.private_storage + import contextlib + @contextlib.contextmanager + def storage(): + nonlocal watcher + with original_storage() as pair: + watcher = pair[1] + yield pair + watcher = None + monkeypatch.setattr(secret_scan, 'private_storage', storage) + monkeypatch.setattr(secret_scan, 'object_batch', kill_watchdog) + assert secret_scan.scan_staged(repo, binary)['reason'] == 'cleanup-watchdog-unavailable' + + +def test_parent_sigkill_triggers_watchdog_cleanup(tmp_path): + import time + script = ('from repo_manager.secret_scan import private_storage; import time\n' + 'with private_storage() as (directory, watcher):\n' + ' (directory / "fixture").write_bytes(b"public generated watchdog fixture")\n' + ' print(str(directory), flush=True)\n' + ' time.sleep(30)\n') + process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + try: + import select + ready, _, _ = select.select([process.stdout], [], [], 5) + assert ready + directory = Path(process.stdout.readline().strip()) + assert directory.is_dir() + process.kill(); process.wait(timeout=5) + until = time.monotonic() + 5 + while directory.exists() and time.monotonic() < until: + time.sleep(.02) + assert not directory.exists() + finally: + if process.poll() is None: + process.kill(); process.wait(timeout=5) + process.stdout.close() + + +def test_watchdog_deadline_quiesces_exact_parent_before_cleanup(): + import select + import time + script = ('from repo_manager import secret_scan as s; import time\n' + 's._CLEANUP = s._CLEANUP.replace("[], [], 180)", "[], [], .2)")\n' + 'with s.private_storage() as (directory, watcher):\n' + ' (directory / "fixture").write_bytes(b"public generated fixture")\n' + ' print(str(directory), flush=True)\n' + ' time.sleep(30)\n') + process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + try: + ready, _, _ = select.select([process.stdout], [], [], 5) + assert ready + directory = Path(process.stdout.readline().strip()) + assert directory.is_dir() + assert process.wait(timeout=5) < 0 + until = time.monotonic() + 5 + while directory.exists() and time.monotonic() < until: + time.sleep(.02) + assert not directory.exists() + finally: + if process.poll() is None: + process.kill(); process.wait(timeout=5) + process.stdout.close() + + +def test_parent_death_kills_source_child(): + import select + script = ('from repo_manager.secret_scan import child_limits\n' + 'import functools, os, subprocess, sys, time\n' + 'child = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(30)"], ' + 'preexec_fn=functools.partial(child_limits, 1024, os.getpid()))\n' + 'print(child.pid, flush=True)\n' + 'time.sleep(30)\n') + process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True) + descriptor = None + try: + ready, _, _ = select.select([process.stdout], [], [], 5) + assert ready + child_pid = int(process.stdout.readline()) + descriptor = os.pidfd_open(child_pid) + process.kill(); process.wait(timeout=5) + dead, _, _ = select.select([descriptor], [], [], 5) + assert dead + finally: + if descriptor is not None: + os.close(descriptor) + if process.poll() is None: + process.kill(); process.wait(timeout=5) + process.stdout.close() diff --git a/workplans/RMGR-WP-0021-secret-detection-controls.md b/workplans/RMGR-WP-0021-secret-detection-controls.md index abbbb9c..e77fc4b 100644 --- a/workplans/RMGR-WP-0021-secret-detection-controls.md +++ b/workplans/RMGR-WP-0021-secret-detection-controls.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: repo-manager created: "2026-10-12" -updated: "2026-10-12" +updated: "2026-10-17" flavor: implementation depends_on: [] state_hub_workstream_id: "5aa6f84a-c682-5996-ae26-96daef670833" @@ -149,3 +149,44 @@ credentials, retain incident owner and revoke/replace/adopt/validate gates. Do n execute founder-deferred rotations or treat suppressed output as remediation. Close only after native coverage and response evidence are adequate for the applicable DSM-DETECT requirements; retain residuals as live work records. + + +## Publish measured staged-scanner batching and verify runtime routing + +```task +id: RMGR-WP-0021-T05 +status: progress +priority: medium +state_hub_task_id: "25fbe694-41d8-5232-bce0-4ed362387d5a" +``` + +2026-10-17 prepared refinement for the local staged-object adapter. Independent +review and a coordinator-executed generated-fixture run cover the exact source +SHA256 `e709c658f779534fcab093207b0a78e49d54fc584c515fdf7ac535ce75f6c400`: +42 tests, 13 exact old/new summary parity cases, immutable scanner snapshot, +volatile-only bounded batches, watchdog and parent-loss refusal/cleanup. Warm +p50 for 1/16/64 distinct 1 KiB blobs is 0.835/0.884/0.904 seconds, compared with +0.702/12.065/49.172 seconds in the retained baseline. The one-blob case regressed; +this is descriptive local fixture evidence, not aggregate workload or tail +acceptance. Larger blobs retain the original stdin route. + +Publish the reviewed patch against the exact owner base, verify the configured +shared and Custodian hooks still resolve to the admitted owner source, then run +clean/private-key/scanner-unavailable generated alternate-index probes without +reading or changing user source/indexes. Verify actual CLI import routing; the +repo-local `.venv/bin/rmgr` imports owner source, but the previously advertised +`/home/worsch/.local/bin/rmgr` is absent in the read-only observation and must not +be described as installed operational coverage. Reconcile that exact CLI route +before claiming CLI publication. Preserve existing hook guards and routing; +retain rollback to the exact prior adapter bytes. Source publication and runtime +acceptance must have separate receipts. + +Review packet: Custodian +`docs/changes/netkingdom-target-2026-10-16/scanner-batch-r1/`. +No canonical owner mutation or runtime admission has occurred in the packet. +This task does not close T03/T04, independent receive/CI coverage, source-control +bypass limits or whole TSA15. + +2026-10-17 integration started: exact reviewed source is applied in the owner +checkout. Canonical tests, configured-hook fixtures and publication are being +verified separately; actual-hook scanner-unavailability remains pending.