684 lines
32 KiB
Diff
684 lines
32 KiB
Diff
--- a/src/repo_manager/secret_scan.py
|
|
+++ b/src/repo_manager/secret_scan.py
|
|
@@ -7,20 +7,42 @@
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
+import contextlib
|
|
+import ctypes
|
|
+import fcntl
|
|
+import functools
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
+import shutil
|
|
+import signal
|
|
+import resource
|
|
import stat
|
|
+import sys
|
|
import subprocess
|
|
import tempfile
|
|
+import time
|
|
from pathlib import Path
|
|
+
|
|
+# Resolve libc and prctl before fork; child_limits performs no library loading.
|
|
+_LIBC = ctypes.CDLL(None, use_errno=True)
|
|
+_PRCTL = _LIBC.prctl
|
|
+_PRCTL.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong]
|
|
+_PRCTL.restype = ctypes.c_int
|
|
|
|
VERSION = '8.30.1'
|
|
BINARY_SHA256 = '88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509'
|
|
MAX_BLOB_BYTES = 8 * 1024 * 1024
|
|
MAX_TOTAL_BYTES = 64 * 1024 * 1024
|
|
MAX_BLOBS = 500
|
|
+MAX_BATCH_BLOBS = 64
|
|
+MAX_BATCH_BYTES = 8 * 1024 * 1024
|
|
+# PIPE_BUF bytes fit the original stdin writer's single atomic write. Larger
|
|
+# blobs retain the stdin path, avoiding a change to fragment boundaries.
|
|
+DIRECTORY_BLOB_BYTES = 4096
|
|
+MAX_REPORT_BYTES = 1024 * 1024
|
|
+SCAN_SECONDS = 120
|
|
CONFIG = '[extend]\nuseDefault = true\n'
|
|
|
|
|
|
@@ -31,7 +53,8 @@
|
|
def git(repo: Path, *args: str) -> bytes:
|
|
env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
|
|
result = subprocess.run(['git', '-C', str(repo), *args], stdout=subprocess.PIPE,
|
|
- stderr=subprocess.DEVNULL, env=env, timeout=20, check=False)
|
|
+ stderr=subprocess.DEVNULL, env=env, timeout=20, check=False,
|
|
+ preexec_fn=functools.partial(child_limits, MAX_TOTAL_BYTES, os.getpid()))
|
|
if result.returncode:
|
|
raise Unverified('git-read-failed')
|
|
return result.stdout
|
|
@@ -63,10 +86,167 @@
|
|
return snapshot, sorted(objects)
|
|
|
|
|
|
+# Source bytes must never be copied onto persistent storage. The watchdog owns
|
|
+# no source data; its pipe EOF detects parent death without a service or hook.
|
|
+_CLEANUP = """
|
|
+import os, select, shutil, signal, stat, sys
|
|
+path, dev, ino, uid, parent = sys.argv[1:]
|
|
+root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
+if os.getppid() != int(parent):
|
|
+ sys.exit(4)
|
|
+pid = os.pidfd_open(int(parent))
|
|
+if os.getppid() != int(parent):
|
|
+ sys.exit(4)
|
|
+os.write(1, b'R')
|
|
+ready, _, _ = select.select([0, pid], [], [], 180)
|
|
+# Exceptional abort targets only the exact parent, never a process group.
|
|
+# If kernel quiescence stalls, retain this observer/duty until death is proven.
|
|
+if not ready:
|
|
+ try:
|
|
+ signal.pidfd_send_signal(pid, signal.SIGTERM)
|
|
+ except ProcessLookupError:
|
|
+ pass
|
|
+ dead, _, _ = select.select([pid], [], [], 5)
|
|
+ if not dead:
|
|
+ try:
|
|
+ signal.pidfd_send_signal(pid, signal.SIGKILL)
|
|
+ except ProcessLookupError:
|
|
+ pass
|
|
+ select.select([pid], [], [])
|
|
+name = os.path.basename(path)
|
|
+try:
|
|
+ info = os.stat(name, dir_fd=root, follow_symlinks=False)
|
|
+except FileNotFoundError:
|
|
+ sys.exit(0)
|
|
+if (info.st_dev, info.st_ino, info.st_uid) != (int(dev), int(ino), int(uid)):
|
|
+ sys.exit(3)
|
|
+if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
|
|
+ sys.exit(3)
|
|
+shutil.rmtree(name, dir_fd=root)
|
|
+"""
|
|
+
|
|
+
|
|
+def is_tmpfs(directory: Path) -> bool:
|
|
+ descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
+ try:
|
|
+ result = ctypes.create_string_buffer(256)
|
|
+ if _LIBC.fstatfs(descriptor, ctypes.byref(result)):
|
|
+ raise OSError(ctypes.get_errno(), 'statfs failed')
|
|
+ return ctypes.c_long.from_buffer(result).value == 0x01021994
|
|
+ finally:
|
|
+ os.close(descriptor)
|
|
+
|
|
+
|
|
+def remove_storage(directory: Path, identity: os.stat_result) -> None:
|
|
+ root = os.open('/dev/shm', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
+ try:
|
|
+ try:
|
|
+ current = os.stat(directory.name, dir_fd=root, follow_symlinks=False)
|
|
+ except FileNotFoundError:
|
|
+ return
|
|
+ if ((current.st_dev, current.st_ino, current.st_uid) !=
|
|
+ (identity.st_dev, identity.st_ino, identity.st_uid)
|
|
+ or not stat.S_ISDIR(current.st_mode)
|
|
+ or stat.S_IMODE(current.st_mode) != 0o700):
|
|
+ raise Unverified('volatile-storage-identity-changed')
|
|
+ shutil.rmtree(directory.name, dir_fd=root)
|
|
+ finally:
|
|
+ os.close(root)
|
|
+
|
|
+
|
|
+@contextlib.contextmanager
|
|
+def private_storage():
|
|
+ if not is_tmpfs(Path('/dev/shm')):
|
|
+ raise Unverified('volatile-storage-unavailable')
|
|
+ temporary = tempfile.mkdtemp(prefix='rmgr-secret-scan-', dir='/dev/shm')
|
|
+ directory = Path(temporary)
|
|
+ identity = directory.lstat()
|
|
+ watcher = None
|
|
+ try:
|
|
+ if not is_tmpfs(directory):
|
|
+ raise Unverified('volatile-storage-unavailable')
|
|
+ watcher = subprocess.Popen([sys.executable, '-I', '-c', _CLEANUP, temporary,
|
|
+ str(identity.st_dev), str(identity.st_ino), str(identity.st_uid), str(os.getpid())],
|
|
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
+ env={'PATH': '/usr/bin:/bin', 'LANG': 'C'}, close_fds=True)
|
|
+ import select
|
|
+ ready, _, _ = select.select([watcher.stdout], [], [], 5)
|
|
+ if not ready or watcher.stdout.read(1) != b'R' or watcher.poll() is not None:
|
|
+ raise Unverified('cleanup-watchdog-unavailable')
|
|
+ watcher.stdout.close()
|
|
+ yield directory, watcher
|
|
+ if watcher.poll() is not None:
|
|
+ raise Unverified('cleanup-watchdog-unavailable')
|
|
+ finally:
|
|
+ pending_error = sys.exc_info()[0] is not None
|
|
+ # All synchronous Git/scanner children have exited before yield unwinds.
|
|
+ try:
|
|
+ remove_storage(directory, identity)
|
|
+ finally:
|
|
+ if watcher is not None:
|
|
+ watcher.stdin.close()
|
|
+ watcher.stdout.close()
|
|
+ try:
|
|
+ code = watcher.wait(timeout=5)
|
|
+ except subprocess.TimeoutExpired:
|
|
+ watcher.kill()
|
|
+ code = watcher.wait(timeout=5)
|
|
+ if code and not pending_error:
|
|
+ raise Unverified('cleanup-watchdog-failed')
|
|
+
|
|
+
|
|
+def child_limits(file_bytes: int, parent_pid: int) -> None:
|
|
+ # Parent loss must quiesce every source-reading/writing child. Install the
|
|
+ # signal before any exec and close the fork/prctl race using the exact PID.
|
|
+ if _PRCTL(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent_pid:
|
|
+ os._exit(127)
|
|
+ # Linux adapter already requires /proc/self/fd. Limits apply only to children.
|
|
+ resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
|
|
+ resource.setrlimit(resource.RLIMIT_CPU, (20, 20))
|
|
+ resource.setrlimit(resource.RLIMIT_AS, (8 * 1024**3, 8 * 1024**3))
|
|
+
|
|
+
|
|
+def object_batch(repo: Path, objects: list[str], directory: Path) -> list[tuple[str, bytes]]:
|
|
+ """Bound Git's output before capture, then validate the complete framing."""
|
|
+ request = ''.join(oid + '\n' for oid in objects).encode('ascii')
|
|
+ env = {**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}
|
|
+ with tempfile.TemporaryFile(dir=directory) as output:
|
|
+ result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch'],
|
|
+ input=request, stdout=output, stderr=subprocess.DEVNULL,
|
|
+ env=env, timeout=20, check=False,
|
|
+ preexec_fn=functools.partial(child_limits,
|
|
+ MAX_BATCH_BYTES + len(objects) * 128, os.getpid()))
|
|
+ if result.returncode:
|
|
+ raise Unverified('git-read-failed')
|
|
+ output.seek(0)
|
|
+ blobs = []
|
|
+ for oid in objects:
|
|
+ header = output.readline(128)
|
|
+ match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)\n', header)
|
|
+ if not match or match[1].decode() != oid:
|
|
+ raise Unverified('invalid-object-batch')
|
|
+ size = int(match[2])
|
|
+ if size > MAX_BLOB_BYTES or size > MAX_BATCH_BYTES:
|
|
+ raise Unverified('staged-byte-limit')
|
|
+ content = output.read(size)
|
|
+ if len(content) != size or output.read(1) != b'\n':
|
|
+ raise Unverified('blob-size-mismatch')
|
|
+ algorithm = 'sha1' if len(oid) == 40 else 'sha256'
|
|
+ digest = hashlib.new(algorithm, b'blob ' + str(size).encode() + b'\0' + content)
|
|
+ if digest.hexdigest() != oid:
|
|
+ raise Unverified('blob-digest-mismatch')
|
|
+ blobs.append((oid, content))
|
|
+ if output.read(1):
|
|
+ raise Unverified('invalid-object-batch')
|
|
+ return blobs
|
|
+
|
|
+
|
|
def scan_staged(repo: Path, binary: Path) -> dict:
|
|
"""Return only fixed statuses/counts and Git blob IDs, never finding contents."""
|
|
scanned = 0
|
|
descriptor = None
|
|
+ executable = None
|
|
+ deadline = time.monotonic() + SCAN_SECONDS
|
|
try:
|
|
descriptor = os.open(binary, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
info = os.fstat(descriptor)
|
|
@@ -74,47 +254,149 @@
|
|
or not info.st_mode & 0o111 or info.st_uid not in (0, os.getuid())
|
|
or info.st_size > 64 * 1024 * 1024):
|
|
raise Unverified('scanner-file-refused')
|
|
+ executable = os.memfd_create('rmgr-verified-scanner', os.MFD_ALLOW_SEALING)
|
|
digest = hashlib.sha256()
|
|
+ copied = 0
|
|
while chunk := os.read(descriptor, 1024 * 1024):
|
|
+ copied += len(chunk)
|
|
+ if copied > 64 * 1024 * 1024:
|
|
+ raise Unverified('scanner-file-refused')
|
|
digest.update(chunk)
|
|
+ with memoryview(chunk) as view:
|
|
+ while view:
|
|
+ written = os.write(executable, view)
|
|
+ if written <= 0:
|
|
+ raise Unverified('scanner-file-refused')
|
|
+ view = view[written:]
|
|
if digest.hexdigest() != BINARY_SHA256:
|
|
raise Unverified('scanner-digest-mismatch')
|
|
+ fcntl.fcntl(executable, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW
|
|
+ | fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL)
|
|
snapshot, objects = staged_objects(repo)
|
|
blocked = []
|
|
- total = 0
|
|
- # Do not honor repository-local config/ignore files, comments or ambient
|
|
- # GITLEAKS settings. No scanner report or credential-bearing diagnostic is kept.
|
|
- with tempfile.TemporaryDirectory(prefix='rmgr-secret-scan-') as temporary:
|
|
- directory = Path(temporary)
|
|
+ # Neither source names nor repository config/ignore files enter this
|
|
+ # private directory. Reports contain ONLY neutral blob paths.
|
|
+ with private_storage() as (directory, watcher):
|
|
+ temporary = str(directory)
|
|
config = directory / 'scanner.toml'; config.write_text(CONFIG)
|
|
ignore = directory / '.gitleaksignore'; ignore.write_text('')
|
|
- env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C', 'TMPDIR': temporary}
|
|
- for oid in objects:
|
|
- size = int(git(repo, 'cat-file', '-s', oid))
|
|
- total += size
|
|
- if size < 0 or size > MAX_BLOB_BYTES or total > MAX_TOTAL_BYTES:
|
|
- raise Unverified('staged-byte-limit')
|
|
- content = git(repo, 'cat-file', 'blob', oid)
|
|
- if len(content) != size:
|
|
+ template = directory / 'report.tmpl'
|
|
+ template.write_text('{{range .}}{{.File}}\n{{end}}')
|
|
+ env = {'PATH': '/usr/bin:/bin', 'HOME': temporary, 'LANG': 'C',
|
|
+ 'TMPDIR': temporary, 'GOMAXPROCS': '2'}
|
|
+ # One bounded metadata query replaces a size subprocess per object.
|
|
+ request = ''.join(oid + '\n' for oid in objects).encode('ascii')
|
|
+ with tempfile.TemporaryFile(dir=directory) as output:
|
|
+ result = subprocess.run(['git', '-C', str(repo), 'cat-file', '--batch-check'],
|
|
+ input=request, stdout=output, stderr=subprocess.DEVNULL,
|
|
+ env={**os.environ, 'GIT_NO_REPLACE_OBJECTS': '1'}, timeout=20,
|
|
+ check=False, preexec_fn=functools.partial(child_limits, MAX_BLOBS * 128, os.getpid()))
|
|
+ if result.returncode:
|
|
+ raise Unverified('git-read-failed')
|
|
+ output.seek(0)
|
|
+ metadata = output.read(MAX_BLOBS * 128 + 1).splitlines()
|
|
+ if len(metadata) != len(objects):
|
|
+ raise Unverified('invalid-object-batch')
|
|
+ sizes = []
|
|
+ for oid, line in zip(objects, metadata):
|
|
+ match = re.fullmatch(rb'([a-f0-9]{40}|[a-f0-9]{64}) blob ([0-9]+)', line)
|
|
+ if not match or match[1].decode() != oid:
|
|
+ raise Unverified('invalid-object-batch')
|
|
+ sizes.append(int(match[2]))
|
|
+ if any(size > MAX_BLOB_BYTES for size in sizes) or sum(sizes) > MAX_TOTAL_BYTES:
|
|
+ raise Unverified('staged-byte-limit')
|
|
+
|
|
+ def invoke(mode: str, content: bytes | None = None) -> list[str]:
|
|
+ if watcher.poll() is not None:
|
|
+ raise Unverified('cleanup-watchdog-unavailable')
|
|
+ remaining = deadline - time.monotonic()
|
|
+ if remaining <= 0:
|
|
+ raise Unverified('scan-deadline-exceeded')
|
|
+ report = directory / 'report.paths'
|
|
+ report.unlink(missing_ok=True)
|
|
+ command = [str(binary), mode, '--config', str(config),
|
|
+ '--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow',
|
|
+ '--redact=100', '--no-banner', '--no-color', '--log-level', 'warn',
|
|
+ '--exit-code', '20', '--timeout', '15']
|
|
+ if mode == 'dir':
|
|
+ command += ['--report-format', 'template', '--report-template', str(template), '--report-path', str(report),
|
|
+ '--max-target-megabytes', '0', 'blobs']
|
|
+ with tempfile.TemporaryFile(dir=directory) as diagnostics:
|
|
+ result = subprocess.run(command, executable=f'/proc/self/fd/{executable}',
|
|
+ pass_fds=(executable,), input=content, cwd=directory, env=env,
|
|
+ stdout=subprocess.DEVNULL, stderr=diagnostics,
|
|
+ timeout=min(20, remaining), check=False,
|
|
+ preexec_fn=functools.partial(child_limits, MAX_REPORT_BYTES, os.getpid()))
|
|
+ # The scanner can warn and skip a file without an error exit.
|
|
+ # Refuse ALL warnings/errors, including partial traversal.
|
|
+ diagnostics.seek(0)
|
|
+ messages = diagnostics.read(MAX_REPORT_BYTES + 1)
|
|
+ # v8.30.1 emits exactly this summary for a successful blocked
|
|
+ # scan. Every other warning/error is a refusal, never output.
|
|
+ summary = re.fullmatch(
|
|
+ rb'[0-9]{1,2}:[0-9]{2}(?:AM|PM) WRN leaks found: [1-9][0-9]*\n',
|
|
+ messages)
|
|
+ if (result.returncode not in (0, 20)
|
|
+ or (messages and not (result.returncode == 20 and summary))):
|
|
+ raise Unverified('scanner-failed')
|
|
+ if mode == 'stdin':
|
|
+ return ['stdin'] if result.returncode == 20 else []
|
|
+ if not report.exists() or report.stat().st_size > MAX_REPORT_BYTES:
|
|
+ raise Unverified('scanner-report-refused')
|
|
+ paths = report.read_bytes().splitlines()
|
|
+ known = {('blobs/' + oid).encode(): oid for oid in small}
|
|
+ if any(path not in known for path in paths):
|
|
+ raise Unverified('scanner-report-refused')
|
|
+ findings = sorted({known[path] for path in paths})
|
|
+ if bool(findings) != (result.returncode == 20):
|
|
+ raise Unverified('scanner-report-refused')
|
|
+ return findings
|
|
+
|
|
+ offset = 0
|
|
+ while offset < len(objects):
|
|
+ batch = []
|
|
+ batch_bytes = 0
|
|
+ while offset < len(objects) and len(batch) < MAX_BATCH_BLOBS:
|
|
+ size = sizes[offset]
|
|
+ if batch and batch_bytes + size > MAX_BATCH_BYTES:
|
|
+ break
|
|
+ batch.append(objects[offset]); batch_bytes += size; offset += 1
|
|
+ if watcher.poll() is not None:
|
|
+ raise Unverified('cleanup-watchdog-unavailable')
|
|
+ blobs = object_batch(repo, batch, directory)
|
|
+ # Check metadata again against exact bytes, including same-size
|
|
+ # substitution via corrupt object storage. Hashes are checked above.
|
|
+ expected = dict(zip(objects, sizes))
|
|
+ if any(len(content) != expected[oid] for oid, content in blobs):
|
|
raise Unverified('blob-size-mismatch')
|
|
- command = [str(binary), 'stdin', '--config', str(config),
|
|
- '--gitleaks-ignore-path', str(ignore), '--ignore-gitleaks-allow',
|
|
- '--redact=100', '--no-banner', '--no-color', '--log-level', 'fatal',
|
|
- '--exit-code', '20', '--timeout', '15']
|
|
- result = subprocess.run(command, executable=f'/proc/self/fd/{descriptor}',
|
|
- pass_fds=(descriptor,), input=content, cwd=directory, env=env,
|
|
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
- timeout=20, check=False)
|
|
- if result.returncode not in (0, 20):
|
|
- raise Unverified('scanner-failed')
|
|
- scanned += 1
|
|
- if result.returncode == 20:
|
|
- blocked.append(oid)
|
|
+ payload = directory / 'blobs'; payload.mkdir(mode=0o700)
|
|
+ small = []
|
|
+ try:
|
|
+ for oid, content in blobs:
|
|
+ if len(content) > DIRECTORY_BLOB_BYTES:
|
|
+ if invoke('stdin', content):
|
|
+ blocked.append(oid)
|
|
+ scanned += 1
|
|
+ else:
|
|
+ path = payload / oid
|
|
+ with path.open('xb') as target:
|
|
+ target.write(content)
|
|
+ path.chmod(0o600)
|
|
+ small.append(oid)
|
|
+ if small:
|
|
+ blocked.extend(invoke('dir'))
|
|
+ scanned += len(small)
|
|
+ finally:
|
|
+ for path in payload.iterdir():
|
|
+ path.unlink()
|
|
+ payload.rmdir()
|
|
if git(repo, 'ls-files', '--stage', '-z') != snapshot:
|
|
raise Unverified('index-changed-during-scan')
|
|
+ if time.monotonic() > deadline:
|
|
+ raise Unverified('scan-deadline-exceeded')
|
|
return {'status': 'blocked' if blocked else 'clean', 'scope': 'changed-staged-blobs',
|
|
'scanner_version': VERSION, 'scanner_sha256': BINARY_SHA256,
|
|
- 'scanned_blobs': scanned, 'blocked_blob_ids': blocked}
|
|
+ 'scanned_blobs': scanned, 'blocked_blob_ids': sorted(blocked)}
|
|
except Unverified as error:
|
|
reason = error.args[0]
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
@@ -122,6 +404,8 @@
|
|
finally:
|
|
if descriptor is not None:
|
|
os.close(descriptor)
|
|
+ if executable is not None:
|
|
+ os.close(executable)
|
|
return {'status': 'unverified', 'scope': 'changed-staged-blobs',
|
|
'reason': reason, 'scanned_blobs': scanned}
|
|
|
|
--- a/tests/test_secret_scan.py
|
|
+++ b/tests/test_secret_scan.py
|
|
@@ -182,7 +182,7 @@
|
|
stage(repo, b'clean\n')
|
|
original = secret_scan.subprocess.run
|
|
def broken(command, **kwargs):
|
|
- if len(command) > 1 and command[1] == 'stdin':
|
|
+ if len(command) > 1 and command[1] in ('stdin', 'dir'):
|
|
if failure == 'timeout':
|
|
raise subprocess.TimeoutExpired(command, 20, output=b'synthetic-sensitive-diagnostic')
|
|
return subprocess.CompletedProcess(command, 2)
|
|
@@ -210,3 +210,263 @@
|
|
assert git(repo, 'rev-parse', 'HEAD') == before
|
|
assert bool(private_key.strip() in result.stdout + result.stderr) is False
|
|
assert b'"status": "blocked"' in result.stdout + result.stderr
|
|
+
|
|
+
|
|
+def test_empty_and_deletion_only_index(binary, repo):
|
|
+ assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
|
+ stage(repo, b'clean\n'); git(repo, 'commit', '-qm', 'seed')
|
|
+ assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
|
+ git(repo, 'rm', '-q', 'application.txt')
|
|
+ assert secret_scan.scan_staged(repo, binary)['scanned_blobs'] == 0
|
|
+
|
|
+
|
|
+def test_duplicate_blobs_and_exact_attribution(binary, repo, private_key):
|
|
+ stage(repo, private_key, 'secret.png') # original filename cannot allowlist content
|
|
+ stage(repo, private_key, 'duplicate.txt')
|
|
+ stage(repo, b'clean\n', 'clean.txt')
|
|
+ oid = git(repo, 'rev-parse', ':secret.png').decode().strip()
|
|
+ result = secret_scan.scan_staged(repo, binary)
|
|
+ assert result['status'] == 'blocked'
|
|
+ assert result['scanned_blobs'] == 2
|
|
+ assert result['blocked_blob_ids'] == [oid]
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize('size', [4096, 4097, 100000, 100001])
|
|
+def test_small_batch_and_large_fallback_equivalence(binary, repo, private_key, size):
|
|
+ data = b'\n' * (size - len(private_key)) + private_key
|
|
+ stage(repo, data)
|
|
+ result = secret_scan.scan_staged(repo, binary)
|
|
+ assert result['status'] == 'blocked'
|
|
+ assert result['blocked_blob_ids'] == [git(repo, 'rev-parse', ':application.txt').decode().strip()]
|
|
+
|
|
+
|
|
+def test_file_boundaries_never_synthesize_a_finding(binary, repo, private_key):
|
|
+ cut = private_key.index(b'\n') // 2
|
|
+ stage(repo, private_key[:cut], 'first')
|
|
+ stage(repo, private_key[cut:], 'second')
|
|
+ # Each independent fragment matches the original single-blob stdin outcome;
|
|
+ # concatenation would synthesize the complete private-key opening marker.
|
|
+ result = secret_scan.scan_staged(repo, binary)
|
|
+ assert result['status'] == 'clean'
|
|
+ assert result['scanned_blobs'] == 2
|
|
+
|
|
+
|
|
+def test_binary_and_encoded_fixture_equivalence(binary, repo, private_key, monkeypatch):
|
|
+ import base64
|
|
+ original = secret_scan.DIRECTORY_BLOB_BYTES
|
|
+ for data in [b'\x00\x01\x02\n' + private_key, base64.b64encode(private_key)]:
|
|
+ stage(repo, data)
|
|
+ monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', 0)
|
|
+ stdin = secret_scan.scan_staged(repo, binary)
|
|
+ monkeypatch.setattr(secret_scan, 'DIRECTORY_BLOB_BYTES', original)
|
|
+ batched = secret_scan.scan_staged(repo, binary)
|
|
+ assert batched == stdin
|
|
+ assert batched['status'] == 'blocked'
|
|
+
|
|
+
|
|
+def test_batch_count_bound_and_independent_findings(binary, repo, private_key, monkeypatch):
|
|
+ monkeypatch.setattr(secret_scan, 'MAX_BATCH_BLOBS', 3)
|
|
+ for i in range(7):
|
|
+ stage(repo, (f'fixture {i}\n'.encode() + private_key) if i % 2 else f'clean {i}\n'.encode(), str(i))
|
|
+ result = secret_scan.scan_staged(repo, binary)
|
|
+ expected = sorted(git(repo, 'rev-parse', ':' + str(i)).decode().strip() for i in [1, 3, 5])
|
|
+ assert result['status'] == 'blocked'
|
|
+ assert result['scanned_blobs'] == 7
|
|
+ assert result['blocked_blob_ids'] == expected
|
|
+
|
|
+
|
|
+def test_total_and_object_count_bounds_refuse(binary, repo, monkeypatch):
|
|
+ stage(repo, b'12345', 'a'); stage(repo, b'67890', 'b')
|
|
+ monkeypatch.setattr(secret_scan, 'MAX_TOTAL_BYTES', 9)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-byte-limit'
|
|
+ monkeypatch.setattr(secret_scan, 'MAX_BLOBS', 1)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'staged-blob-limit'
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize('mutation', ['path', 'in-place'])
|
|
+def test_verified_executable_snapshot_survives_mutation(binary, repo, monkeypatch, tmp_path, mutation):
|
|
+ import shutil
|
|
+ scanner = tmp_path / 'scanner'; shutil.copyfile(binary, scanner); scanner.chmod(0o700)
|
|
+ stage(repo, b'clean\n')
|
|
+ original = secret_scan.staged_objects
|
|
+ def mutate(repo):
|
|
+ if mutation == 'path':
|
|
+ scanner.unlink()
|
|
+ scanner.write_bytes(b'not executable scanner bytes\n')
|
|
+ scanner.chmod(0o700)
|
|
+ return original(repo)
|
|
+ monkeypatch.setattr(secret_scan, 'staged_objects', mutate)
|
|
+ assert secret_scan.scan_staged(repo, scanner)['status'] == 'clean'
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize('fault', ['warning', 'unknown-path', 'missing-report', 'returncode'])
|
|
+def test_partial_scans_and_invalid_reports_refuse(binary, repo, monkeypatch, fault):
|
|
+ stage(repo, b'clean\n')
|
|
+ original = secret_scan.subprocess.run
|
|
+ def fail(command, **kwargs):
|
|
+ result = original(command, **kwargs)
|
|
+ if command[1] == 'dir':
|
|
+ report = Path(command[command.index('--report-path') + 1])
|
|
+ if fault == 'warning':
|
|
+ kwargs['stderr'].write(b'synthetic-sensitive-diagnostic\n')
|
|
+ elif fault == 'unknown-path':
|
|
+ report.write_text('blobs/unknown\n')
|
|
+ elif fault == 'missing-report':
|
|
+ report.unlink()
|
|
+ else:
|
|
+ return subprocess.CompletedProcess(command, 20)
|
|
+ return result
|
|
+ monkeypatch.setattr(secret_scan.subprocess, 'run', fail)
|
|
+ result = secret_scan.scan_staged(repo, binary)
|
|
+ assert result['status'] == 'unverified'
|
|
+ assert 'synthetic-sensitive-diagnostic' not in json.dumps(result)
|
|
+
|
|
+
|
|
+def test_deadline_exhaustion_refuses(binary, repo, monkeypatch):
|
|
+ stage(repo, b'clean\n')
|
|
+ monkeypatch.setattr(secret_scan, 'SCAN_SECONDS', 0)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'scan-deadline-exceeded'
|
|
+
|
|
+
|
|
+def test_git_object_output_corruption_refuses(binary, repo, monkeypatch):
|
|
+ stage(repo, b'clean\n')
|
|
+ original = secret_scan.subprocess.run
|
|
+ def corrupt(command, **kwargs):
|
|
+ result = original(command, **kwargs)
|
|
+ if command[-1] == '--batch':
|
|
+ output = kwargs['stdout']
|
|
+ output.seek(-3, 2); output.write(b'xx\n')
|
|
+ return result
|
|
+ monkeypatch.setattr(secret_scan.subprocess, 'run', corrupt)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'blob-digest-mismatch'
|
|
+
|
|
+
|
|
+def test_report_growth_limit_refuses(binary, repo, private_key, monkeypatch):
|
|
+ stage(repo, private_key)
|
|
+ monkeypatch.setattr(secret_scan, 'MAX_REPORT_BYTES', 16)
|
|
+ assert secret_scan.scan_staged(repo, binary)['status'] == 'unverified'
|
|
+
|
|
+
|
|
+def test_sealed_executable_rejects_write(binary, repo, monkeypatch):
|
|
+ import errno
|
|
+ stage(repo, b'clean\n')
|
|
+ original = secret_scan.subprocess.run
|
|
+ checked = []
|
|
+ def assert_sealed(command, **kwargs):
|
|
+ if command[1] in ('dir', 'stdin'):
|
|
+ fd = kwargs['pass_fds'][0]
|
|
+ with pytest.raises(OSError) as error:
|
|
+ os.pwrite(fd, b'X', 0)
|
|
+ assert error.value.errno == errno.EPERM
|
|
+ checked.append(True)
|
|
+ return original(command, **kwargs)
|
|
+ monkeypatch.setattr(secret_scan.subprocess, 'run', assert_sealed)
|
|
+ assert secret_scan.scan_staged(repo, binary)['status'] == 'clean'
|
|
+ assert checked
|
|
+
|
|
+
|
|
+def test_nonvolatile_storage_refuses(binary, repo, monkeypatch):
|
|
+ stage(repo, b'clean\n')
|
|
+ monkeypatch.setattr(secret_scan, 'is_tmpfs', lambda directory: False)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'volatile-storage-unavailable'
|
|
+
|
|
+
|
|
+def test_cleanup_watchdog_loss_refuses(binary, repo, monkeypatch):
|
|
+ stage(repo, b'clean\n')
|
|
+ original = secret_scan.object_batch
|
|
+ def kill_watchdog(repo, objects, directory):
|
|
+ # Discover only our child watchdog using the context manager wrapper.
|
|
+ watcher.kill(); watcher.wait(timeout=5)
|
|
+ return original(repo, objects, directory)
|
|
+ original_storage = secret_scan.private_storage
|
|
+ import contextlib
|
|
+ @contextlib.contextmanager
|
|
+ def storage():
|
|
+ nonlocal watcher
|
|
+ with original_storage() as pair:
|
|
+ watcher = pair[1]
|
|
+ yield pair
|
|
+ watcher = None
|
|
+ monkeypatch.setattr(secret_scan, 'private_storage', storage)
|
|
+ monkeypatch.setattr(secret_scan, 'object_batch', kill_watchdog)
|
|
+ assert secret_scan.scan_staged(repo, binary)['reason'] == 'cleanup-watchdog-unavailable'
|
|
+
|
|
+
|
|
+def test_parent_sigkill_triggers_watchdog_cleanup(tmp_path):
|
|
+ import time
|
|
+ script = ('from repo_manager.secret_scan import private_storage; import time\n'
|
|
+ 'with private_storage() as (directory, watcher):\n'
|
|
+ ' (directory / "fixture").write_bytes(b"public generated watchdog fixture")\n'
|
|
+ ' print(str(directory), flush=True)\n'
|
|
+ ' time.sleep(30)\n')
|
|
+ process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
|
+ stderr=subprocess.DEVNULL, text=True)
|
|
+ try:
|
|
+ import select
|
|
+ ready, _, _ = select.select([process.stdout], [], [], 5)
|
|
+ assert ready
|
|
+ directory = Path(process.stdout.readline().strip())
|
|
+ assert directory.is_dir()
|
|
+ process.kill(); process.wait(timeout=5)
|
|
+ until = time.monotonic() + 5
|
|
+ while directory.exists() and time.monotonic() < until:
|
|
+ time.sleep(.02)
|
|
+ assert not directory.exists()
|
|
+ finally:
|
|
+ if process.poll() is None:
|
|
+ process.kill(); process.wait(timeout=5)
|
|
+ process.stdout.close()
|
|
+
|
|
+
|
|
+def test_watchdog_deadline_quiesces_exact_parent_before_cleanup():
|
|
+ import select
|
|
+ import time
|
|
+ script = ('from repo_manager import secret_scan as s; import time\n'
|
|
+ 's._CLEANUP = s._CLEANUP.replace("[], [], 180)", "[], [], .2)")\n'
|
|
+ 'with s.private_storage() as (directory, watcher):\n'
|
|
+ ' (directory / "fixture").write_bytes(b"public generated fixture")\n'
|
|
+ ' print(str(directory), flush=True)\n'
|
|
+ ' time.sleep(30)\n')
|
|
+ process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
|
+ stderr=subprocess.DEVNULL, text=True)
|
|
+ try:
|
|
+ ready, _, _ = select.select([process.stdout], [], [], 5)
|
|
+ assert ready
|
|
+ directory = Path(process.stdout.readline().strip())
|
|
+ assert directory.is_dir()
|
|
+ assert process.wait(timeout=5) < 0
|
|
+ until = time.monotonic() + 5
|
|
+ while directory.exists() and time.monotonic() < until:
|
|
+ time.sleep(.02)
|
|
+ assert not directory.exists()
|
|
+ finally:
|
|
+ if process.poll() is None:
|
|
+ process.kill(); process.wait(timeout=5)
|
|
+ process.stdout.close()
|
|
+
|
|
+
|
|
+def test_parent_death_kills_source_child():
|
|
+ import select
|
|
+ script = ('from repo_manager.secret_scan import child_limits\n'
|
|
+ 'import functools, os, subprocess, sys, time\n'
|
|
+ 'child = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(30)"], '
|
|
+ 'preexec_fn=functools.partial(child_limits, 1024, os.getpid()))\n'
|
|
+ 'print(child.pid, flush=True)\n'
|
|
+ 'time.sleep(30)\n')
|
|
+ process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE,
|
|
+ stderr=subprocess.DEVNULL, text=True)
|
|
+ descriptor = None
|
|
+ try:
|
|
+ ready, _, _ = select.select([process.stdout], [], [], 5)
|
|
+ assert ready
|
|
+ child_pid = int(process.stdout.readline())
|
|
+ descriptor = os.pidfd_open(child_pid)
|
|
+ process.kill(); process.wait(timeout=5)
|
|
+ dead, _, _ = select.select([descriptor], [], [], 5)
|
|
+ assert dead
|
|
+ finally:
|
|
+ if descriptor is not None:
|
|
+ os.close(descriptor)
|
|
+ if process.poll() is None:
|
|
+ process.kill(); process.wait(timeout=5)
|
|
+ process.stdout.close()
|