perf(security): batch small staged blobs with bounded volatile storage
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1468b-63fa-7710-85dc-8f2e16b3ebfa
This commit is contained in:
parent
97ea248925
commit
83c521bf2f
3 changed files with 615 additions and 30 deletions
|
|
@ -182,7 +182,7 @@ def test_scanner_failure_never_passes_or_emits_diagnostics(binary, repo, monkeyp
|
|||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def broken(command, **kwargs):
|
||||
if len(command) > 1 and command[1] == 'stdin':
|
||||
if len(command) > 1 and command[1] in ('stdin', 'dir'):
|
||||
if failure == 'timeout':
|
||||
raise subprocess.TimeoutExpired(command, 20, output=b'synthetic-sensitive-diagnostic')
|
||||
return subprocess.CompletedProcess(command, 2)
|
||||
|
|
@ -210,3 +210,263 @@ def test_actual_commit_only_hook_scans_its_temporary_index(binary, repo, private
|
|||
assert git(repo, 'rev-parse', 'HEAD') == before
|
||||
assert bool(private_key.strip() in result.stdout + result.stderr) is False
|
||||
assert b'"status": "blocked"' in result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_empty_and_deletion_only_index(binary, repo):
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
stage(repo, b'clean\n'); git(repo, 'commit', '-qm', 'seed')
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
git(repo, 'rm', '-q', 'application.txt')
|
||||
assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
||||
|
||||
|
||||
def test_duplicate_blobs_and_exact_attribution(binary, repo, private_key):
|
||||
stage(repo, private_key, 'secret.png') # original filename cannot allowlist content
|
||||
stage(repo, private_key, 'duplicate.txt')
|
||||
stage(repo, b'clean\n', 'clean.txt')
|
||||
oid = git(repo, 'rev-parse', ':secret.png').decode().strip()
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['scanned_blobs'] == 2
|
||||
assert result['blocked_blob_ids'] == [oid]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size', [4096, 4097, 100000, 100001])
|
||||
def test_small_batch_and_large_fallback_equivalence(binary, repo, private_key, size):
|
||||
data = b'\n' * (size - len(private_key)) + private_key
|
||||
stage(repo, data)
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['blocked_blob_ids'] == [git(repo, 'rev-parse', ':application.txt').decode().strip()]
|
||||
|
||||
|
||||
def test_file_boundaries_never_synthesize_a_finding(binary, repo, private_key):
|
||||
cut = private_key.index(b'\n') // 2
|
||||
stage(repo, private_key[:cut], 'first')
|
||||
stage(repo, private_key[cut:], 'second')
|
||||
# Each independent fragment matches the original single-blob stdin outcome;
|
||||
# concatenation would synthesize the complete private-key opening marker.
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'clean'
|
||||
assert result['scanned_blobs'] == 2
|
||||
|
||||
|
||||
def test_binary_and_encoded_fixture_equivalence(binary, repo, private_key, monkeypatch):
|
||||
import base64
|
||||
original = secret_scan.DIRECTORY_BLOB_BYTES
|
||||
for data in [b'\x00\x01\x02\n' + private_key, base64.b64encode(private_key)]:
|
||||
stage(repo, data)
|
||||
monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', 0)
|
||||
stdin = secret_scan.scan_staged(repo, binary)
|
||||
monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', original)
|
||||
batched = secret_scan.scan_staged(repo, binary)
|
||||
assert batched == stdin
|
||||
assert batched['status'] == 'blocked'
|
||||
|
||||
|
||||
def test_batch_count_bound_and_independent_findings(binary, repo, private_key, monkeypatch):
|
||||
monkeypatch.setattr(secret_scan, 'MAX_BATCH_BLOBS', 3)
|
||||
for i in range(7):
|
||||
stage(repo, (f'fixture {i}\n'.encode() + private_key) if i % 2 else f'clean {i}\n'.encode(), str(i))
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
expected = sorted(git(repo, 'rev-parse', ':' + str(i)).decode().strip() for i in [1, 3, 5])
|
||||
assert result['status'] == 'blocked'
|
||||
assert result['scanned_blobs'] == 7
|
||||
assert result['blocked_blob_ids'] == expected
|
||||
|
||||
|
||||
def test_total_and_object_count_bounds_refuse(binary, repo, monkeypatch):
|
||||
stage(repo, b'12345', 'a'); stage(repo, b'67890', 'b')
|
||||
monkeypatch.setattr(secret_scan, 'MAX_TOTAL_BYTES', 9)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-byte-limit'
|
||||
monkeypatch.setattr(secret_scan, 'MAX_BLOBS', 1)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-blob-limit'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mutation', ['path', 'in-place'])
|
||||
def test_verified_executable_snapshot_survives_mutation(binary, repo, monkeypatch, tmp_path, mutation):
|
||||
import shutil
|
||||
scanner = tmp_path / 'scanner'; shutil.copyfile(binary, scanner); scanner.chmod(0o700)
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.staged_objects
|
||||
def mutate(repo):
|
||||
if mutation == 'path':
|
||||
scanner.unlink()
|
||||
scanner.write_bytes(b'not executable scanner bytes\n')
|
||||
scanner.chmod(0o700)
|
||||
return original(repo)
|
||||
monkeypatch.setattr(secret_scan, 'staged_objects', mutate)
|
||||
assert secret_scan.scan_staged(repo, scanner)['status'] == 'clean'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fault', ['warning', 'unknown-path', 'missing-report', 'returncode'])
|
||||
def test_partial_scans_and_invalid_reports_refuse(binary, repo, monkeypatch, fault):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def fail(command, **kwargs):
|
||||
result = original(command, **kwargs)
|
||||
if command[1] == 'dir':
|
||||
report = Path(command[command.index('--report-path') + 1])
|
||||
if fault == 'warning':
|
||||
kwargs['stderr'].write(b'synthetic-sensitive-diagnostic\n')
|
||||
elif fault == 'unknown-path':
|
||||
report.write_text('blobs/unknown\n')
|
||||
elif fault == 'missing-report':
|
||||
report.unlink()
|
||||
else:
|
||||
return subprocess.CompletedProcess(command, 20)
|
||||
return result
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', fail)
|
||||
result = secret_scan.scan_staged(repo, binary)
|
||||
assert result['status'] == 'unverified'
|
||||
assert 'synthetic-sensitive-diagnostic' not in json.dumps(result)
|
||||
|
||||
|
||||
def test_deadline_exhaustion_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
monkeypatch.setattr(secret_scan, 'SCAN_SECONDS', 0)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'scan-deadline-exceeded'
|
||||
|
||||
|
||||
def test_git_object_output_corruption_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
def corrupt(command, **kwargs):
|
||||
result = original(command, **kwargs)
|
||||
if command[-1] == '--batch':
|
||||
output = kwargs['stdout']
|
||||
output.seek(-3, 2); output.write(b'xx\n')
|
||||
return result
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', corrupt)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'blob-digest-mismatch'
|
||||
|
||||
|
||||
def test_report_growth_limit_refuses(binary, repo, private_key, monkeypatch):
|
||||
stage(repo, private_key)
|
||||
monkeypatch.setattr(secret_scan, 'MAX_REPORT_BYTES', 16)
|
||||
assert secret_scan.scan_staged(repo, binary)['status'] == 'unverified'
|
||||
|
||||
|
||||
def test_sealed_executable_rejects_write(binary, repo, monkeypatch):
|
||||
import errno
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.subprocess.run
|
||||
checked = []
|
||||
def assert_sealed(command, **kwargs):
|
||||
if command[1] in ('dir', 'stdin'):
|
||||
fd = kwargs['pass_fds'][0]
|
||||
with pytest.raises(OSError) as error:
|
||||
os.pwrite(fd, b'X', 0)
|
||||
assert error.value.errno == errno.EPERM
|
||||
checked.append(True)
|
||||
return original(command, **kwargs)
|
||||
monkeypatch.setattr(secret_scan.subprocess, 'run', assert_sealed)
|
||||
assert secret_scan.scan_staged(repo, binary)['status'] == 'clean'
|
||||
assert checked
|
||||
|
||||
|
||||
def test_nonvolatile_storage_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
monkeypatch.setattr(secret_scan, 'is_tmpfs', lambda directory: False)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'volatile-storage-unavailable'
|
||||
|
||||
|
||||
def test_cleanup_watchdog_loss_refuses(binary, repo, monkeypatch):
|
||||
stage(repo, b'clean\n')
|
||||
original = secret_scan.object_batch
|
||||
def kill_watchdog(repo, objects, directory):
|
||||
# Discover only our child watchdog using the context manager wrapper.
|
||||
watcher.kill(); watcher.wait(timeout=5)
|
||||
return original(repo, objects, directory)
|
||||
original_storage = secret_scan.private_storage
|
||||
import contextlib
|
||||
@contextlib.contextmanager
|
||||
def storage():
|
||||
nonlocal watcher
|
||||
with original_storage() as pair:
|
||||
watcher = pair[1]
|
||||
yield pair
|
||||
watcher = None
|
||||
monkeypatch.setattr(secret_scan, 'private_storage', storage)
|
||||
monkeypatch.setattr(secret_scan, 'object_batch', kill_watchdog)
|
||||
assert secret_scan.scan_staged(repo, binary)['reason'] == 'cleanup-watchdog-unavailable'
|
||||
|
||||
|
||||
def test_parent_sigkill_triggers_watchdog_cleanup(tmp_path):
|
||||
import time
|
||||
script = ('from repo_manager.secret_scan import private_storage; import time\n'
|
||||
'with private_storage() as (directory, watcher):\n'
|
||||
' (directory / "fixture").write_bytes(b"public generated watchdog fixture")\n'
|
||||
' print(str(directory), flush=True)\n'
|
||||
' time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
try:
|
||||
import select
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
directory = Path(process.stdout.readline().strip())
|
||||
assert directory.is_dir()
|
||||
process.kill(); process.wait(timeout=5)
|
||||
until = time.monotonic() + 5
|
||||
while directory.exists() and time.monotonic() < until:
|
||||
time.sleep(.02)
|
||||
assert not directory.exists()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
||||
|
||||
def test_watchdog_deadline_quiesces_exact_parent_before_cleanup():
|
||||
import select
|
||||
import time
|
||||
script = ('from repo_manager import secret_scan as s; import time\n'
|
||||
's._CLEANUP = s._CLEANUP.replace("[], [], 180)", "[], [], .2)")\n'
|
||||
'with s.private_storage() as (directory, watcher):\n'
|
||||
' (directory / "fixture").write_bytes(b"public generated fixture")\n'
|
||||
' print(str(directory), flush=True)\n'
|
||||
' time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
try:
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
directory = Path(process.stdout.readline().strip())
|
||||
assert directory.is_dir()
|
||||
assert process.wait(timeout=5) < 0
|
||||
until = time.monotonic() + 5
|
||||
while directory.exists() and time.monotonic() < until:
|
||||
time.sleep(.02)
|
||||
assert not directory.exists()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
||||
|
||||
def test_parent_death_kills_source_child():
|
||||
import select
|
||||
script = ('from repo_manager.secret_scan import child_limits\n'
|
||||
'import functools, os, subprocess, sys, time\n'
|
||||
'child = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(30)"], '
|
||||
'preexec_fn=functools.partial(child_limits, 1024, os.getpid()))\n'
|
||||
'print(child.pid, flush=True)\n'
|
||||
'time.sleep(30)\n')
|
||||
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True)
|
||||
descriptor = None
|
||||
try:
|
||||
ready, _, _ = select.select([process.stdout], [], [], 5)
|
||||
assert ready
|
||||
child_pid = int(process.stdout.readline())
|
||||
descriptor = os.pidfd_open(child_pid)
|
||||
process.kill(); process.wait(timeout=5)
|
||||
dead, _, _ = select.select([descriptor], [], [], 5)
|
||||
assert dead
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if process.poll() is None:
|
||||
process.kill(); process.wait(timeout=5)
|
||||
process.stdout.close()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue