perf(security): batch small staged blobs with bounded volatile storage
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1468b-63fa-7710-85dc-8f2e16b3ebfa
This commit is contained in:
parent
97ea248925
commit
83c521bf2f
3 changed files with 615 additions and 30 deletions
|
|
@ -7,20 +7,42 @@ claim history/push/CI coverage. Its caller must treat unverified as refusal.
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import ctypes
|
||||
import fcntl
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import resource
|
||||
import stat
|
||||
import sys
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Resolve libc and prctl before fork; child_limits performs no library loading.
|
||||
_LIBC = ctypes.CDLL(None, use_errno=True)
|
||||
_PRCTL = _LIBC.prctl
|
||||
_PRCTL.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong]
|
||||
_PRCTL.restype = ctypes.c_int
|
||||
|
||||
VERSION = '8.30.1'
|
||||
BINARY_SHA256 = '88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509'
|
||||
MAX_BLOB_BYTES = 8 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 64 * 1024 * 1024
|
||||
MAX_BLOBS = 500
|
||||
MAX_BATCH_BLOBS = 64
|
||||
MAX_BATCH_BYTES = 8 * 1024 * 1024
|
||||
# PIPE_BUF bytes fit the original stdin writer's single atomic write. Larger
|
||||
# blobs retain the stdin path, avoiding a change to fragment boundaries.
|
||||
DIRECTORY_BLOB_BYTES = 4096
|
||||
MAX_REPORT_BYTES = 1024 * 1024
|
||||
SCAN_SECONDS = 120
|
||||
CONFIG = '[extend]\nuseDefault = true\n'
|
||||
|
||||
|
||||
|
|
@ -31,7 +53,8 @@ class Unverified(Exception):
|
|||
def git(repo: Path, *args: str) -> bytes:
|
||||
env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
|
||||
result = subprocess.run(['git', '-C', str(repo), *args], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, env=env, timeout=20, check=False)
|
||||
stderr=subprocess.DEVNULL, env=env, timeout=20, check=False,
|
||||
preexec_fn=functools.partial(child_limits, MAX_TOTAL_BYTES, os.getpid()))
|
||||
if result.returncode:
|
||||
raise Unverified('git-read-failed')
|
||||
return result.stdout
|
||||
|
|
@ -63,10 +86,167 @@ def staged_objects(repo: Path) -> tuple[bytes, list[str]]:
|
|||
return snapshot, sorted(objects)
|
||||
|
||||
|
||||
# Source bytes must never be copied onto persistent storage. The watchdog owns
|
||||
# no source data; its pipe EOF detects parent death without a service or hook.
|
||||
_CLEANUP = """
|
||||
import os, select, shutil, signal, stat, sys
|
||||
path, dev, ino, uid, parent = sys.argv[1:]
|
||||
root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
if os.getppid() != int(parent):
|
||||
sys.exit(4)
|
||||
pid = os.pidfd_open(int(parent))
|
||||
if os.getppid() != int(parent):
|
||||
sys.exit(4)
|
||||
os.write(1, b'R')
|
||||
ready, _, _ = select.select([0, pid], [], [], 180)
|
||||
# Exceptional abort targets only the exact parent, never a process group.
|
||||
# If kernel quiescence stalls, retain this observer/duty until death is proven.
|
||||
if not ready:
|
||||
try:
|
||||
signal.pidfd_send_signal(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
dead, _, _ = select.select([pid], [], [], 5)
|
||||
if not dead:
|
||||
try:
|
||||
signal.pidfd_send_signal(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
select.select([pid], [], [])
|
||||
name = os.path.basename(path)
|
||||
try:
|
||||
info = os.stat(name, dir_fd=root, follow_symlinks=False)
|
||||
except FileNotFoundError:
|
||||
sys.exit(0)
|
||||
if (info.st_dev, info.st_ino, info.st_uid) != (int(dev), int(ino), int(uid)):
|
||||
sys.exit(3)
|
||||
if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
|
||||
sys.exit(3)
|
||||
shutil.rmtree(name, dir_fd=root)
|
||||
"""
|
||||
|
||||
|
||||
def is_tmpfs(directory: Path) -> bool:
|
||||
descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
result = ctypes.create_string_buffer(256)
|
||||
if _LIBC.fstatfs(descriptor, ctypes.byref(result)):
|
||||
raise OSError(ctypes.get_errno(), 'statfs failed')
|
||||
return ctypes.c_long.from_buffer(result).value == 0x01021994
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def remove_storage(directory: Path, identity: os.stat_result) -> None:
|
||||
root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try:
|
||||
try:
|
||||
current = os.stat(directory.name, dir_fd=root, follow_symlinks=False)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if ((current.st_dev, current.st_ino, current.st_uid) !=
|
||||
(identity.st_dev, identity.st_ino, identity.st_uid)
|
||||
or not stat.S_ISDIR(current.st_mode)
|
||||
or stat.S_IMODE(current.st_mode) != 0o700):
|
||||
raise Unverified('volatile-storage-identity-changed')
|
||||
shutil.rmtree(directory.name, dir_fd=root)
|
||||
finally:
|
||||
os.close(root)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def private_storage():
|
||||
if not is_tmpfs(Path('/dev/shm')):
|
||||
raise Unverified('volatile-storage-unavailable')
|
||||
temporary = tempfile.mkdtemp(prefix='rmgr-secret-scan-', dir='/dev/shm')
|
||||
directory = Path(temporary)
|
||||
identity = directory.lstat()
|
||||
watcher = None
|
||||
try:
|
||||
if not is_tmpfs(directory):
|
||||
raise Unverified('volatile-storage-unavailable')
|
||||
watcher = subprocess.Popen([sys.executable, '-I', '-c', _CLEANUP, temporary,
|
||||
str(identity.st_dev), str(identity.st_ino), str(identity.st_uid), str(os.getpid())],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
||||
env={'PATH': '/usr/bin:/bin', 'LANG': 'C'}, close_fds=True)
|
||||
import select
|
||||
ready, _, _ = select.select([watcher.stdout], [], [], 5)
|
||||
if not ready or watcher.stdout.read(1) != b'R' or watcher.poll() is not None:
|
||||
raise Unverified('cleanup-watchdog-unavailable')
|
||||
watcher.stdout.close()
|
||||
yield directory, watcher
|
||||
if watcher.poll() is not None:
|
||||
raise Unverified('cleanup-watchdog-unavailable')
|
||||
finally:
|
||||
pending_error = sys.exc_info()[0] is not None
|
||||
# All synchronous Git/scanner children have exited before yield unwinds.
|
||||
try:
|
||||
remove_storage(directory, identity)
|
||||
finally:
|
||||
if watcher is not None:
|
||||
watcher.stdin.close()
|
||||
watcher.stdout.close()
|
||||
try:
|
||||
code = watcher.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
watcher.kill()
|
||||
code = watcher.wait(timeout=5)
|
||||
if code and not pending_error:
|
||||
raise Unverified('cleanup-watchdog-failed')
|
||||
|
||||
|
||||
def child_limits(file_bytes: int, parent_pid: int) -> None:
|
||||
# Parent loss must quiesce every source-reading/writing child. Install the
|
||||
# signal before any exec and close the fork/prctl race using the exact PID.
|
||||
if _PRCTL(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent_pid:
|
||||
os._exit(127)
|
||||
# Linux adapter already requires /proc/self/fd. Limits apply only to children.
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (20, 20))
|
||||
resource.setrlimit(resource.RLIMIT_AS, (8 * 1024**3, 8 * 1024**3))
|
||||
|
||||
|
||||
def object_batch(repo: Path, objects: list[str], directory: Path) -> list[tuple[str, bytes]]:
|
||||
"""Bound Git's output before capture, then validate the complete framing."""
|
||||
request = ''.join(oid + '\n' for oid in objects).encode('ascii')
|
||||
env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
|
||||
with tempfile.TemporaryFile(dir=directory) as output:
|
||||
result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch'],
|
||||
input=request, stdout=output, stderr=subprocess.DEVNULL,
|
||||
env=env, timeout=20, check=False,
|
||||
preexec_fn=functools.partial(child_limits,
|
||||
MAX_BATCH_BYTES + len(objects) * 128, os.getpid()))
|
||||
if result.returncode:
|
||||
raise Unverified('git-read-failed')
|
||||
output.seek(0)
|
||||
blobs = []
|
||||
for oid in objects:
|
||||
header = output.readline(128)
|
||||
match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)\n', header)
|
||||
if not match or match[1].decode() != oid:
|
||||
raise Unverified('invalid-object-batch')
|
||||
size = int(match[2])
|
||||
if size > MAX_BLOB_BYTES or size > MAX_BATCH_BYTES:
|
||||
raise Unverified('staged-byte-limit')
|
||||
content = output.read(size)
|
||||
if len(content) != size or output.read(1) != b'\n':
|
||||
raise Unverified('blob-size-mismatch')
|
||||
algorithm = 'sha1' if len(oid) == 40 else 'sha256'
|
||||
digest = hashlib.new(algorithm, b'blob ' + str(size).encode() + b'\0' + content)
|
||||
if digest.hexdigest() != oid:
|
||||
raise Unverified('blob-digest-mismatch')
|
||||
blobs.append((oid, content))
|
||||
if output.read(1):
|
||||
raise Unverified('invalid-object-batch')
|
||||
return blobs
|
||||
|
||||
|
||||
def scan_staged(repo: Path, binary: Path) -> dict:
|
||||
"""Return only fixed statuses/counts and Git blob IDs, never finding contents."""
|
||||
scanned = 0
|
||||
descriptor = None
|
||||
executable = None
|
||||
deadline = time.monotonic() + SCAN_SECONDS
|
||||
try:
|
||||
descriptor = os.open(binary, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
info = os.fstat(descriptor)
|
||||
|
|
@ -74,47 +254,149 @@ def scan_staged(repo: Path, binary: Path) -> dict:
|
|||
or not info.st_mode & 0o111 or info.st_uid not in (0, os.getuid())
|
||||
or info.st_size > 64 * 1024 * 1024):
|
||||
raise Unverified('scanner-file-refused')
|
||||
executable = os.memfd_create('rmgr-verified-scanner', os.MFD_ALLOW_SEALING)
|
||||
digest = hashlib.sha256()
|
||||
copied = 0
|
||||
while chunk := os.read(descriptor, 1024 * 1024):
|
||||
copied += len(chunk)
|
||||
if copied > 64 * 1024 * 1024:
|
||||
raise Unverified('scanner-file-refused')
|
||||
digest.update(chunk)
|
||||
with memoryview(chunk) as view:
|
||||
while view:
|
||||
written = os.write(executable, view)
|
||||
if written <= 0:
|
||||
raise Unverified('scanner-file-refused')
|
||||
view = view[written:]
|
||||
if digest.hexdigest() != BINARY_SHA256:
|
||||
raise Unverified('scanner-digest-mismatch')
|
||||
fcntl.fcntl(executable, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW
|
||||
| fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL)
|
||||
snapshot, objects = staged_objects(repo)
|
||||
blocked = []
|
||||
total = 0
|
||||
# Do not honor repository-local config/ignore files, comments or ambient
|
||||
# GITLEAKS settings. No scanner report or credential-bearing diagnostic is kept.
|
||||
with tempfile.TemporaryDirectory(prefix='rmgr-secret-scan-') as temporary:
|
||||
directory = Path(temporary)
|
||||
# Neither source names nor repository config/ignore files enter this
|
||||
# private directory. Reports contain ONLY neutral blob paths.
|
||||
with private_storage() as (directory, watcher):
|
||||
temporary = str(directory)
|
||||
config = directory / 'scanner.toml'; config.write_text(CONFIG)
|
||||
ignore = directory / '.gitleaksignore'; ignore.write_text('')
|
||||
env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C', 'TMPDIR': temporary}
|
||||
for oid in objects:
|
||||
size = int(git(repo, 'cat-file', '-s', oid))
|
||||
total += size
|
||||
if size < 0 or size > MAX_BLOB_BYTES or total > MAX_TOTAL_BYTES:
|
||||
raise Unverified('staged-byte-limit')
|
||||
content = git(repo, 'cat-file', 'blob', oid)
|
||||
if len(content) != size:
|
||||
template = directory / 'report.tmpl'
|
||||
template.write_text('{{range .}}{{.File}}\n{{end}}')
|
||||
env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C',
|
||||
'TMPDIR': temporary, 'GOMAXPROCS': '2'}
|
||||
# One bounded metadata query replaces a size subprocess per object.
|
||||
request = ''.join(oid + '\n' for oid in objects).encode('ascii')
|
||||
with tempfile.TemporaryFile(dir=directory) as output:
|
||||
result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch-check'],
|
||||
input=request, stdout=output, stderr=subprocess.DEVNULL,
|
||||
env={**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}, timeout=20,
|
||||
check=False, preexec_fn=functools.partial(child_limits, MAX_BLOBS * 128, os.getpid()))
|
||||
if result.returncode:
|
||||
raise Unverified('git-read-failed')
|
||||
output.seek(0)
|
||||
metadata = output.read(MAX_BLOBS * 128 + 1).splitlines()
|
||||
if len(metadata) != len(objects):
|
||||
raise Unverified('invalid-object-batch')
|
||||
sizes = []
|
||||
for oid, line in zip(objects, metadata):
|
||||
match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)', line)
|
||||
if not match or match[1].decode() != oid:
|
||||
raise Unverified('invalid-object-batch')
|
||||
sizes.append(int(match[2]))
|
||||
if any(size > MAX_BLOB_BYTES for size in sizes) or sum(sizes) > MAX_TOTAL_BYTES:
|
||||
raise Unverified('staged-byte-limit')
|
||||
|
||||
def invoke(mode: str, content: bytes | None = None) -> list[str]:
|
||||
if watcher.poll() is not None:
|
||||
raise Unverified('cleanup-watchdog-unavailable')
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise Unverified('scan-deadline-exceeded')
|
||||
report = directory / 'report.paths'
|
||||
report.unlink(missing_ok=True)
|
||||
command = [str(binary), mode, '--config', str(config),
|
||||
'--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow',
|
||||
'--redact=100', '--no-banner', '--no-color', '--log-level', 'warn',
|
||||
'--exit-code', '20', '--timeout', '15']
|
||||
if mode == 'dir':
|
||||
command += ['--report-format', 'template', '--report-template', str(template), '--report-path', str(report),
|
||||
'--max-target-megabytes', '0', 'blobs']
|
||||
with tempfile.TemporaryFile(dir=directory) as diagnostics:
|
||||
result = subprocess.run(command, executable=f'/proc/self/fd/{executable}',
|
||||
pass_fds=(executable,), input=content, cwd=directory, env=env,
|
||||
stdout=subprocess.DEVNULL, stderr=diagnostics,
|
||||
timeout=min(20, remaining), check=False,
|
||||
preexec_fn=functools.partial(child_limits, MAX_REPORT_BYTES, os.getpid()))
|
||||
# The scanner can warn and skip a file without an error exit.
|
||||
# Refuse ALL warnings/errors, including partial traversal.
|
||||
diagnostics.seek(0)
|
||||
messages = diagnostics.read(MAX_REPORT_BYTES + 1)
|
||||
# v8.30.1 emits exactly this summary for a successful blocked
|
||||
# scan. Every other warning/error is a refusal, never output.
|
||||
summary = re.fullmatch(
|
||||
rb'[0-9]{1,2}:[0-9]{2}(?:AM|PM) WRN leaks found: [1-9][0-9]*\n',
|
||||
messages)
|
||||
if (result.returncode not in (0, 20)
|
||||
or (messages and not (result.returncode == 20 and summary))):
|
||||
raise Unverified('scanner-failed')
|
||||
if mode == 'stdin':
|
||||
return ['stdin'] if result.returncode == 20 else []
|
||||
if not report.exists() or report.stat().st_size > MAX_REPORT_BYTES:
|
||||
raise Unverified('scanner-report-refused')
|
||||
paths = report.read_bytes().splitlines()
|
||||
known = {('blobs/' + oid).encode(): oid for oid in small}
|
||||
if any(path not in known for path in paths):
|
||||
raise Unverified('scanner-report-refused')
|
||||
findings = sorted({known[path] for path in paths})
|
||||
if bool(findings) != (result.returncode == 20):
|
||||
raise Unverified('scanner-report-refused')
|
||||
return findings
|
||||
|
||||
offset = 0
|
||||
while offset < len(objects):
|
||||
batch = []
|
||||
batch_bytes = 0
|
||||
while offset < len(objects) and len(batch) < MAX_BATCH_BLOBS:
|
||||
size = sizes[offset]
|
||||
if batch and batch_bytes + size > MAX_BATCH_BYTES:
|
||||
break
|
||||
batch.append(objects[offset]); batch_bytes += size; offset += 1
|
||||
if watcher.poll() is not None:
|
||||
raise Unverified('cleanup-watchdog-unavailable')
|
||||
blobs = object_batch(repo, batch, directory)
|
||||
# Check metadata again against exact bytes, including same-size
|
||||
# substitution via corrupt object storage. Hashes are checked above.
|
||||
expected = dict(zip(objects, sizes))
|
||||
if any(len(content) != expected[oid] for oid, content in blobs):
|
||||
raise Unverified('blob-size-mismatch')
|
||||
command = [str(binary), 'stdin', '--config', str(config),
|
||||
'--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow',
|
||||
'--redact=100', '--no-banner', '--no-color', '--log-level', 'fatal',
|
||||
'--exit-code', '20', '--timeout', '15']
|
||||
result = subprocess.run(command, executable=f'/proc/self/fd/{descriptor}',
|
||||
pass_fds=(descriptor,), input=content, cwd=directory, env=env,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=20, check=False)
|
||||
if result.returncode not in (0, 20):
|
||||
raise Unverified('scanner-failed')
|
||||
scanned += 1
|
||||
if result.returncode == 20:
|
||||
blocked.append(oid)
|
||||
payload = directory / 'blobs'; payload.mkdir(mode=0o700)
|
||||
small = []
|
||||
try:
|
||||
for oid, content in blobs:
|
||||
if len(content) > DIRECTORY_BLOB_BYTES:
|
||||
if invoke('stdin', content):
|
||||
blocked.append(oid)
|
||||
scanned += 1
|
||||
else:
|
||||
path = payload / oid
|
||||
with path.open('xb') as target:
|
||||
target.write(content)
|
||||
path.chmod(0o600)
|
||||
small.append(oid)
|
||||
if small:
|
||||
blocked.extend(invoke('dir'))
|
||||
scanned += len(small)
|
||||
finally:
|
||||
for path in payload.iterdir():
|
||||
path.unlink()
|
||||
payload.rmdir()
|
||||
if git(repo, 'ls-files', '--stage', '-z') != snapshot:
|
||||
raise Unverified('index-changed-during-scan')
|
||||
if time.monotonic() > deadline:
|
||||
raise Unverified('scan-deadline-exceeded')
|
||||
return {'status': 'blocked' if blocked else 'clean', 'scope': 'changed-staged-blobs',
|
||||
'scanner_version': VERSION, 'scanner_sha256': BINARY_SHA256,
|
||||
'scanned_blobs': scanned, 'blocked_blob_ids': blocked}
|
||||
'scanned_blobs': scanned, 'blocked_blob_ids': sorted(blocked)}
|
||||
except Unverified as error:
|
||||
reason = error.args[0]
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
|
|
@ -122,6 +404,8 @@ def scan_staged(repo: Path, binary: Path) -> dict:
|
|||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if executable is not None:
|
||||
os.close(executable)
|
||||
return {'status': 'unverified', 'scope': 'changed-staged-blobs',
|
||||
'reason': reason, 'scanned_blobs': scanned}
|
||||
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ def test_scanner_failure_never_passes_or_emits_diagnostics(binary, repo, monkeyp
|
|||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def broken(command, **kwargs):
|
||||
if len(command) > 1 and command[1] == 'stdin':
|
||||
if len(command) > 1 and command[1] in ('stdin', 'dir'):
|
||||
if failure == 'timeout':
|
||||
raise subprocess.TimeoutExpired(command, 20, output=b'synthetic-sensitive-diagnostic')
|
||||
return subprocess.CompletedProcess(command, 2)
|
||||
|
|
@ -210,3 +210,263 @@ def test_actual_commit_only_hook_scans_its_temporary_index(binary, repo, private
|
|||
assert git(repo, 'rev-parse', 'HEAD') == before
|
||||
assert bool(private_key.strip() in result.stdout + result.stderr) is False
|
||||
assert b'"status": "blocked"' in result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_empty_and_deletion_only_index(binary, repo):
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
stage(repo, b'clean\n'); git(repo, 'commit', '-qm', 'seed')
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
git(repo, 'rm', '-q', 'application.txt')
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
|
||||
|
||||
def test_duplicate_blobs_and_exact_attribution(binary, repo, private_key):
|
||||
stage(repo, private_key, 'secret.png') # original filename cannot allowlist content
|
||||
stage(repo, private_key, 'duplicate.txt')
|
||||
stage(repo, b'clean\n', 'clean.txt')
|
||||
oid = git(repo, 'rev-parse', ':secret.png').decode().strip()
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['scanned_blobs'] == 2
|
||||
assert result['blocked_blob_ids'] == [oid]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size', [4096, 4097, 100000, 100001])
|
||||
def test_small_batch_and_large_fallback_equivalence(binary, repo, private_key, size):
|
||||
data = b'\n' * (size - len(private_key)) + private_key
|
||||
stage(repo, data)
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['blocked_blob_ids'] == [git(repo, 'rev-parse', ':application.txt').decode().strip()]
|
||||
|
||||
|
||||
def test_file_boundaries_never_synthesize_a_finding(binary, repo, private_key):
|
||||
cut = private_key.index(b'\n') // 2
|
||||
stage(repo, private_key[:cut], 'first')
|
||||
stage(repo, private_key[cut:], 'second')
|
||||
# Each independent fragment matches the original single-blob stdin outcome;
|
||||
# concatenation would synthesize the complete private-key opening marker.
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'clean'
|
||||
assert result['scanned_blobs'] == 2
|
||||
|
||||
|
||||
def test_binary_and_encoded_fixture_equivalence(binary, repo, private_key, monkeypatch):
|
||||
import base64
|
||||
original = secret_scan.DIRECTORY_BLOB_BYTES
|
||||
for data in [b'\x00\x01\x02\n' + private_key, base64.b64encode(private_key)]:
|
||||
stage(repo, data)
|
||||
monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', 0)
|
||||
stdin = secret_scan.scan_staged(repo, binary)
|
||||
monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', original)
|
||||
batched = secret_scan.scan_staged(repo, binary)
|
||||
assert batched == stdin
|
||||
assert batched['status'] == 'blocked'
|
||||
|
||||
|
||||
def test_batch_count_bound_and_independent_findings(binary, repo, private_key, monkeypatch):
|
||||
monkeypatch.setattr(secret_scan, 'MAX_BATCH_BLOBS', 3)
|
||||
for i in range(7):
|
||||
stage(repo, (f'fixture {i}\n'.encode() + private_key) if i % 2 else f'clean {i}\n'.encode(), str(i))
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
expected = sorted(git(repo, 'rev-parse', ':' + str(i)).decode().strip() for i in [1, 3, 5])
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['scanned_blobs'] == 7
|
||||
assert result['blocked_blob_ids'] == expected
|
||||
|
||||
|
||||
def test_total_and_object_count_bounds_refuse(binary, repo, monkeypatch):
|
||||
stage(repo, b'12345', 'a'); stage(repo, b'67890', 'b')
|
||||
monkeypatch.setattr(secret_scan, 'MAX_TOTAL_BYTES', 9)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-byte-limit'
|
||||
monkeypatch.setattr(secret_scan, 'MAX_BLOBS', 1)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-blob-limit'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mutation', ['path', 'in-place'])
|
||||
def test_verified_executable_snapshot_survives_mutation(binary, repo, monkeypatch, tmp_path, mutation):
|
||||
import shutil
|
||||
scanner = tmp_path / 'scanner'; shutil.copyfile(binary, scanner); scanner.chmod(0o700)
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.staged_objects
|
||||
def mutate(repo):
|
||||
if mutation == 'path':
|
||||
scanner.unlink()
|
||||
scanner.write_bytes(b'not executable scanner bytes\n')
|
||||
scanner.chmod(0o700)
|
||||
return original(repo)
|
||||
monkeypatch.setattr(secret_scan, 'staged_objects', mutate)
|
||||
assert secret_scan.scan_staged(repo, scanner)['status'] == 'clean'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fault', ['warning', 'unknown-path', 'missing-report', 'returncode'])
|
||||
def test_partial_scans_and_invalid_reports_refuse(binary, repo, monkeypatch, fault):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def fail(command, **kwargs):
|
||||
result = original(command, **kwargs)
|
||||
if command[1] == 'dir':
|
||||
report = Path(command[command.index('--report-path') + 1])
|
||||
if fault == 'warning':
|
||||
kwargs['stderr'].write(b'synthetic-sensitive-diagnostic\n')
|
||||
elif fault == 'unknown-path':
|
||||
report.write_text('blobs/unknown\n')
|
||||
elif fault == 'missing-report':
|
||||
report.unlink()
|
||||
else:
|
||||
return subprocess.CompletedProcess(command, 20)
|
||||
return result
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', fail)
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'unverified'
|
||||
assert 'synthetic-sensitive-diagnostic' not in json.dumps(result)
|
||||
|
||||
|
||||
def test_deadline_exhaustion_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
monkeypatch.setattr(secret_scan, 'SCAN_SECONDS', 0)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'scan-deadline-exceeded'
|
||||
|
||||
|
||||
def test_git_object_output_corruption_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def corrupt(command, **kwargs):
|
||||
result = original(command, **kwargs)
|
||||
if command[-1] == '--batch':
|
||||
output = kwargs['stdout']
|
||||
output.seek(-3, 2); output.write(b'xx\n')
|
||||
return result
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', corrupt)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'blob-digest-mismatch'
|
||||
|
||||
|
||||
def test_report_growth_limit_refuses(binary, repo, private_key, monkeypatch):
|
||||
stage(repo, private_key)
|
||||
monkeypatch.setattr(secret_scan, 'MAX_REPORT_BYTES', 16)
|
||||
assert secret_scan.scan_staged(repo, binary)['status'] == 'unverified'
|
||||
|
||||
|
||||
def test_sealed_executable_rejects_write(binary, repo, monkeypatch):
|
||||
import errno
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
checked = []
|
||||
def assert_sealed(command, **kwargs):
|
||||
if command[1] in ('dir', 'stdin'):
|
||||
fd = kwargs['pass_fds'][0]
|
||||
with pytest.raises(OSError) as error:
|
||||
os.pwrite(fd, b'X', 0)
|
||||
assert error.value.errno == errno.EPERM
|
||||
checked.append(True)
|
||||
return original(command, **kwargs)
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', assert_sealed)
|
||||
assert secret_scan.scan_staged(repo, binary)['status'] == 'clean'
|
||||
assert checked
|
||||
|
||||
|
||||
def test_nonvolatile_storage_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
monkeypatch.setattr(secret_scan, 'is_tmpfs', lambda directory: False)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'volatile-storage-unavailable'
|
||||
|
||||
|
||||
def test_cleanup_watchdog_loss_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.object_batch
|
||||
def kill_watchdog(repo, objects, directory):
|
||||
# Discover only our child watchdog using the context manager wrapper.
|
||||
watcher.kill(); watcher.wait(timeout=5)
|
||||
return original(repo, objects, directory)
|
||||
original_storage = secret_scan.private_storage
|
||||
import contextlib
|
||||
@contextlib.contextmanager
|
||||
def storage():
|
||||
nonlocal watcher
|
||||
with original_storage() as pair:
|
||||
watcher = pair[1]
|
||||
yield pair
|
||||
watcher = None
|
||||
monkeypatch.setattr(secret_scan, 'private_storage', storage)
|
||||
monkeypatch.setattr(secret_scan, 'object_batch', kill_watchdog)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'cleanup-watchdog-unavailable'
|
||||
|
||||
|
||||
def test_parent_sigkill_triggers_watchdog_cleanup(tmp_path):
|
||||
import time
|
||||
script = ('from repo_manager.secret_scan import private_storage; import time\n'
|
||||
'with private_storage() as (directory, watcher):\n'
|
||||
' (directory / "fixture").write_bytes(b"public generated watchdog fixture")\n'
|
||||
' print(str(directory), flush=True)\n'
|
||||
' time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
try:
|
||||
import select
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
directory = Path(process.stdout.readline().strip())
|
||||
assert directory.is_dir()
|
||||
process.kill(); process.wait(timeout=5)
|
||||
until = time.monotonic() + 5
|
||||
while directory.exists() and time.monotonic() < until:
|
||||
time.sleep(.02)
|
||||
assert not directory.exists()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
||||
|
||||
def test_watchdog_deadline_quiesces_exact_parent_before_cleanup():
|
||||
import select
|
||||
import time
|
||||
script = ('from repo_manager import secret_scan as s; import time\n'
|
||||
's._CLEANUP = s._CLEANUP.replace("[], [], 180)", "[], [], .2)")\n'
|
||||
'with s.private_storage() as (directory, watcher):\n'
|
||||
' (directory / "fixture").write_bytes(b"public generated fixture")\n'
|
||||
' print(str(directory), flush=True)\n'
|
||||
' time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
try:
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
directory = Path(process.stdout.readline().strip())
|
||||
assert directory.is_dir()
|
||||
assert process.wait(timeout=5) < 0
|
||||
until = time.monotonic() + 5
|
||||
while directory.exists() and time.monotonic() < until:
|
||||
time.sleep(.02)
|
||||
assert not directory.exists()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
||||
|
||||
def test_parent_death_kills_source_child():
|
||||
import select
|
||||
script = ('from repo_manager.secret_scan import child_limits\n'
|
||||
'import functools, os, subprocess, sys, time\n'
|
||||
'child = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(30)"], '
|
||||
'preexec_fn=functools.partial(child_limits, 1024, os.getpid()))\n'
|
||||
'print(child.pid, flush=True)\n'
|
||||
'time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
descriptor = None
|
||||
try:
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
child_pid = int(process.stdout.readline())
|
||||
descriptor = os.pidfd_open(child_pid)
|
||||
process.kill(); process.wait(timeout=5)
|
||||
dead, _, _ = select.select([descriptor], [], [], 5)
|
||||
assert dead
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ status: active
|
|||
owner: codex
|
||||
topic_slug: repo-manager
|
||||
created: "2026-10-12"
|
||||
updated: "2026-10-12"
|
||||
updated: "2026-10-17"
|
||||
flavor: implementation
|
||||
depends_on: []
|
||||
state_hub_workstream_id: "5aa6f84a-c682-5996-ae26-96daef670833"
|
||||
|
|
@ -149,3 +149,44 @@ credentials, retain incident owner and revoke/replace/adopt/validate gates. Do n
|
|||
execute founder-deferred rotations or treat suppressed output as remediation.
|
||||
Close only after native coverage and response evidence are adequate for the
|
||||
applicable DSM-DETECT requirements; retain residuals as live work records.
|
||||
|
||||
|
||||
## Publish measured staged-scanner batching and verify runtime routing
|
||||
|
||||
```task
|
||||
id: RMGR-WP-0021-T05
|
||||
status: progress
|
||||
priority: medium
|
||||
state_hub_task_id: "25fbe694-41d8-5232-bce0-4ed362387d5a"
|
||||
```
|
||||
|
||||
2026-10-17 prepared refinement for the local staged-object adapter. Independent
|
||||
review and a coordinator-executed generated-fixture run cover the exact source
|
||||
SHA256 `e709c658f779534fcab093207b0a78e49d54fc584c515fdf7ac535ce75f6c400`:
|
||||
42 tests, 13 exact old/new summary parity cases, immutable scanner snapshot,
|
||||
volatile-only bounded batches, watchdog and parent-loss refusal/cleanup. Warm
|
||||
p50 for 1/16/64 distinct 1 KiB blobs is 0.835/0.884/0.904 seconds, compared with
|
||||
0.702/12.065/49.172 seconds in the retained baseline. The one-blob case regressed;
|
||||
this is descriptive local fixture evidence, not aggregate workload or tail
|
||||
acceptance. Larger blobs retain the original stdin route.
|
||||
|
||||
Publish the reviewed patch against the exact owner base, verify the configured
|
||||
shared and Custodian hooks still resolve to the admitted owner source, then run
|
||||
clean/private-key/scanner-unavailable generated alternate-index probes without
|
||||
reading or changing user source/indexes. Verify actual CLI import routing; the
|
||||
repo-local `.venv/bin/rmgr` imports owner source, but the previously advertised
|
||||
`/home/worsch/.local/bin/rmgr` is absent in the read-only observation and must not
|
||||
be described as installed operational coverage. Reconcile that exact CLI route
|
||||
before claiming CLI publication. Preserve existing hook guards and routing;
|
||||
retain rollback to the exact prior adapter bytes. Source publication and runtime
|
||||
acceptance must have separate receipts.
|
||||
|
||||
Review packet: Custodian
|
||||
`docs/changes/netkingdom-target-2026-10-16/scanner-batch-r1/`.
|
||||
No canonical owner mutation or runtime admission has occurred in the packet.
|
||||
This task does not close T03/T04, independent receive/CI coverage, source-control
|
||||
bypass limits or whole TSA15.
|
||||
|
||||
2026-10-17 integration started: exact reviewed source is applied in the owner
|
||||
checkout. Canonical tests, configured-hook fixtures and publication are being
|
||||
verified separately; actual-hook scanner-unavailability remains pending.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue