tegwick
ea65606c12
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a14bfc-9643-7872-aa12-98326dcf25e7
481 lines
14 KiB
Go
481 lines
14 KiB
Go
// receive-secret-scan is a content-silent, Linux pre-receive control adapter.
|
|
// It never changes refs, runs repository programs, or retrieves credentials.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const scannerHash = "88f91962aa2f93ac6ab281d553b9e125f5197bbbce38f9f2437f7299c32e5509"
|
|
const scannerVersion = "8.30.1"
|
|
const maxInput = 512 * 1024
|
|
const maxObject = 8 * 1024 * 1024
|
|
const maxTotal = 64 * 1024 * 1024
|
|
const maxObjects = 2000
|
|
const maxRefs = 10000
|
|
|
|
var oidPattern = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
|
var errLimit = errors.New("output-limit")
|
|
|
|
type result struct {
|
|
Status string `json:"status"`
|
|
Scope string `json:"scope"`
|
|
Reason string `json:"reason,omitempty"`
|
|
Scanned int `json:"scanned_objects"`
|
|
ScannedRefs int `json:"scanned_refs"`
|
|
BlockedID string `json:"blocked_object_id,omitempty"`
|
|
Version string `json:"scanner_version"`
|
|
}
|
|
type update struct{ Old, New, Ref string }
|
|
type cappedBuffer struct {
|
|
bytes.Buffer
|
|
limit int
|
|
}
|
|
|
|
func (b *cappedBuffer) Write(p []byte) (int, error) {
|
|
if len(p) > b.limit-b.Len() {
|
|
return 0, errLimit
|
|
}
|
|
return b.Buffer.Write(p)
|
|
}
|
|
func zero(s string) bool { return strings.Trim(s, "0") == "" }
|
|
func parseUpdates(raw []byte) ([]update, error) {
|
|
if len(raw) == 0 || len(raw) > maxInput || raw[len(raw)-1] != '\n' {
|
|
return nil, errors.New("invalid-update-input")
|
|
}
|
|
lines := strings.Split(string(raw[:len(raw)-1]), "\n")
|
|
if len(lines) > 256 {
|
|
return nil, errors.New("update-limit")
|
|
}
|
|
seen := map[string]bool{}
|
|
var out []update
|
|
for _, line := range lines {
|
|
fields := strings.Split(line, " ")
|
|
if len(fields) != 3 || !oidPattern.MatchString(fields[0]) || !oidPattern.MatchString(fields[1]) || len(fields[0]) != len(fields[1]) || len(fields[2]) > 1024 || !strings.HasPrefix(fields[2], "refs/") || strings.ContainsAny(fields[2], "\x00\r\n\t") || seen[fields[2]] || zero(fields[0]) && zero(fields[1]) {
|
|
return nil, errors.New("invalid-update-input")
|
|
}
|
|
seen[fields[2]] = true
|
|
out = append(out, update{fields[0], fields[1], fields[2]})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Only receive-pack object-location variables cross into Git. Scanner processes
|
|
// get none. Never inherit arbitrary GIT_CONFIG_* or credential/provider settings.
|
|
func gitEnvironment() []string {
|
|
env := []string{"PATH=/usr/local/bin:/usr/bin:/bin", "HOME=/nonexistent", "LANG=C", "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_NO_REPLACE_OBJECTS=1", "GIT_GRAFT_FILE=/dev/null", "GIT_NO_LAZY_FETCH=1", "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/false", "GIT_SSH_COMMAND=/bin/false"}
|
|
for _, key := range []string{"GIT_DIR", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_QUARANTINE_PATH"} {
|
|
if val, ok := os.LookupEnv(key); ok {
|
|
env = append(env, key+"="+val)
|
|
}
|
|
}
|
|
return env
|
|
}
|
|
func git(ctx context.Context, input []byte, limit int, args ...string) ([]byte, error) {
|
|
cmd := exec.CommandContext(ctx, "/usr/bin/git", args...)
|
|
cmd.Env = gitEnvironment()
|
|
cmd.Stdin = bytes.NewReader(input)
|
|
b := &cappedBuffer{limit: limit}
|
|
cmd.Stdout = b
|
|
cmd.Stderr = io.Discard
|
|
if runChild(cmd) != nil {
|
|
return nil, errors.New("git-" + args[0] + "-failed")
|
|
}
|
|
return b.Bytes(), nil
|
|
}
|
|
func refs(ctx context.Context) ([]byte, map[string]string, error) {
|
|
raw, err := git(ctx, nil, 2*1024*1024, "for-each-ref", "--format=%(objectname) %(refname)")
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
found := map[string]string{}
|
|
for _, line := range strings.Split(strings.TrimSuffix(string(raw), "\n"), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
parts := strings.Split(line, " ")
|
|
if len(parts) != 2 || !oidPattern.MatchString(parts[0]) || !strings.HasPrefix(parts[1], "refs/") {
|
|
return nil, nil, errors.New("invalid-ref-snapshot")
|
|
}
|
|
found[parts[1]] = parts[0]
|
|
}
|
|
if len(found) > maxRefs {
|
|
return nil, nil, errors.New("ref-limit")
|
|
}
|
|
return raw, found, nil
|
|
}
|
|
func pinScanner(path string) (*os.File, error) {
|
|
fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return nil, errors.New("scanner-file-refused")
|
|
}
|
|
f := os.NewFile(uintptr(fd), path)
|
|
ok := false
|
|
defer func() {
|
|
if !ok {
|
|
f.Close()
|
|
}
|
|
}()
|
|
if err = verifyScannerFD(f); err != nil {
|
|
return nil, err
|
|
}
|
|
ok = true
|
|
return f, nil
|
|
}
|
|
func scan(ctx context.Context, f *os.File, dir string, content []byte) (bool, error) {
|
|
cmd := exec.CommandContext(ctx, "/proc/self/fd/3", "stdin", "--config", filepath.Join(dir, "scanner.toml"), "--gitleaks-ignore-path", filepath.Join(dir, "ignore"), "--ignore-gitleaks-allow", "--redact=100", "--no-banner", "--no-color", "--log-level", "fatal", "--exit-code", "20", "--timeout", "15")
|
|
cmd.ExtraFiles = []*os.File{f}
|
|
cmd.Dir = dir
|
|
cmd.Env = []string{"PATH=/usr/bin:/bin", "HOME=" + dir, "TMPDIR=" + dir, "LANG=C"}
|
|
cmd.Stdin = bytes.NewReader(content)
|
|
cmd.Stdout = io.Discard
|
|
cmd.Stderr = io.Discard
|
|
err := runChild(cmd)
|
|
if ctx.Err() != nil {
|
|
return false, errors.New("scan-deadline-exceeded")
|
|
}
|
|
if err == nil {
|
|
return false, nil
|
|
}
|
|
var exit *exec.ExitError
|
|
if errors.As(err, &exit) && exit.ExitCode() == 20 {
|
|
return true, nil
|
|
}
|
|
return false, errors.New("scanner-failed")
|
|
}
|
|
|
|
// Tree object names are covered too. Gitlinks refer to an external repository,
|
|
// so this version refuses them rather than claiming transitive coverage.
|
|
func treeContent(raw []byte, oidBytes int) ([]byte, error) {
|
|
var names []byte
|
|
for len(raw) > 0 {
|
|
space := bytes.IndexByte(raw, ' ')
|
|
nul := bytes.IndexByte(raw, 0)
|
|
if space <= 0 || nul <= space || len(raw) < nul+1+oidBytes {
|
|
return nil, errors.New("invalid-tree")
|
|
}
|
|
mode := string(raw[:space])
|
|
if mode == "160000" {
|
|
return nil, errors.New("external-gitlink-unverified")
|
|
}
|
|
if mode != "100644" && mode != "100755" && mode != "120000" && mode != "40000" {
|
|
return nil, errors.New("unsupported-tree-entry")
|
|
}
|
|
names = append(names, raw[space+1:nul]...)
|
|
names = append(names, '\n')
|
|
raw = raw[nul+1+oidBytes:]
|
|
}
|
|
return names, nil
|
|
}
|
|
func check(ctx context.Context, updates []update, binary string) (out result) {
|
|
out = result{Status: "unverified", Scope: "introduced-reachable-git-objects-and-updated-ref-names", Version: scannerVersion}
|
|
fail := func(err error) result { out.Reason = err.Error(); return out }
|
|
scanner, err := pinScanner(binary)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
defer scanner.Close()
|
|
format, err := git(ctx, nil, 128, "rev-parse", "--show-object-format")
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
width := 40
|
|
if string(format) == "sha256\n" {
|
|
width = 64
|
|
} else if string(format) != "sha1\n" {
|
|
return fail(errors.New("unsupported-object-format"))
|
|
}
|
|
shallow, err := git(ctx, nil, 128, "rev-parse", "--is-shallow-repository")
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
if string(shallow) != "false\n" {
|
|
return fail(errors.New("shallow-repository-unverified"))
|
|
}
|
|
// Partial repositories may perform network retrieval while reading objects.
|
|
config, err := git(ctx, nil, 1024*1024, "config", "--local", "--includes", "--name-only", "--list")
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
for _, key := range strings.Split(strings.ToLower(string(config)), "\n") {
|
|
if key == "extensions.partialclone" || strings.HasSuffix(key, ".promisor") {
|
|
return fail(errors.New("partial-repository-unverified"))
|
|
}
|
|
}
|
|
snapshot, existing, err := refs(ctx)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
tips := map[string]bool{}
|
|
for _, u := range updates {
|
|
if len(u.Old) != width {
|
|
return fail(errors.New("object-format-mismatch"))
|
|
}
|
|
if _, err = git(ctx, nil, 128, "check-ref-format", u.Ref); err != nil {
|
|
return fail(errors.New("invalid-ref-name"))
|
|
}
|
|
current, exists := existing[u.Ref]
|
|
if zero(u.Old) && exists || !zero(u.Old) && current != u.Old {
|
|
return fail(errors.New("old-ref-mismatch"))
|
|
}
|
|
if !zero(u.New) {
|
|
tips[u.New] = true
|
|
}
|
|
}
|
|
// Freeze exclusions from the pre-push snapshot, never a mutable --all read.
|
|
// Scan all introduced reachable objects, not just a tip diff or first parent.
|
|
var revisions strings.Builder
|
|
for tip := range tips {
|
|
revisions.WriteString(tip + "\n")
|
|
}
|
|
objects := map[string]bool{}
|
|
if len(tips) > 0 {
|
|
excluded := map[string]bool{}
|
|
for _, oid := range existing {
|
|
excluded[oid] = true
|
|
}
|
|
for oid := range excluded {
|
|
revisions.WriteString("^" + oid + "\n")
|
|
}
|
|
raw, e := git(ctx, []byte(revisions.String()), (maxObjects+1)*(width+1), "rev-list", "--objects", "--no-object-names", "--stdin")
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
for _, oid := range strings.Fields(string(raw)) {
|
|
if !oidPattern.MatchString(oid) || len(oid) != width {
|
|
return fail(errors.New("invalid-object-list"))
|
|
}
|
|
objects[oid] = true
|
|
}
|
|
}
|
|
if len(objects) > maxObjects {
|
|
return fail(errors.New("object-limit"))
|
|
}
|
|
ids := make([]string, 0, len(objects))
|
|
for oid := range objects {
|
|
ids = append(ids, oid)
|
|
}
|
|
sort.Strings(ids)
|
|
var metadata []byte
|
|
if len(ids) > 0 {
|
|
metadata, err = git(ctx, []byte(strings.Join(ids, "\n")+"\n"), (maxObjects+1)*100, "cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)")
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
}
|
|
lines := strings.Split(strings.TrimSuffix(string(metadata), "\n"), "\n")
|
|
if len(ids) == 0 {
|
|
lines = nil
|
|
}
|
|
if len(lines) != len(ids) {
|
|
return fail(errors.New("incomplete-object-metadata"))
|
|
}
|
|
// Reject oversized batches before reading/scanning any object payload.
|
|
type objectInfo struct {
|
|
kind string
|
|
size int64
|
|
}
|
|
var objectMetadata []objectInfo
|
|
total := int64(0)
|
|
for i, line := range lines {
|
|
parts := strings.Split(line, " ")
|
|
if len(parts) != 3 || parts[0] != ids[i] {
|
|
return fail(errors.New("invalid-object-metadata"))
|
|
}
|
|
kind := parts[1]
|
|
if kind != "blob" && kind != "commit" && kind != "tag" && kind != "tree" {
|
|
return fail(errors.New("unsupported-object-type"))
|
|
}
|
|
size, e := strconv.ParseInt(parts[2], 10, 64)
|
|
if e != nil || size < 0 || size > maxObject || total+size > maxTotal {
|
|
return fail(errors.New("object-byte-limit"))
|
|
}
|
|
total += size
|
|
objectMetadata = append(objectMetadata, objectInfo{kind, size})
|
|
}
|
|
dir, err := os.MkdirTemp("", "forge-secret-scan-")
|
|
if err != nil {
|
|
return fail(errors.New("temporary-directory-unavailable"))
|
|
}
|
|
defer os.RemoveAll(dir)
|
|
var storage *volatileStorage
|
|
defer func() {
|
|
if storage != nil {
|
|
if e := storage.close(); e != nil {
|
|
out.Status = "unverified"
|
|
out.Reason = e.Error()
|
|
}
|
|
}
|
|
}()
|
|
getStorage := func() error {
|
|
if storage == nil {
|
|
var e error
|
|
storage, e = newVolatileStorage()
|
|
return e
|
|
}
|
|
if !storage.live() {
|
|
return errors.New("cleanup-watchdog-unavailable")
|
|
}
|
|
return nil
|
|
}
|
|
if os.WriteFile(filepath.Join(dir, "scanner.toml"), []byte("[extend]\nuseDefault = true\n"), 0600) != nil || os.WriteFile(filepath.Join(dir, "ignore"), nil, 0600) != nil {
|
|
return fail(errors.New("scanner-config-unavailable"))
|
|
}
|
|
// Ref names can themselves contain recognized tokens. Never echo them.
|
|
for start := 0; start < len(updates); start += directoryBatchInputs {
|
|
end := start + directoryBatchInputs
|
|
if end > len(updates) {
|
|
end = len(updates)
|
|
}
|
|
if end-start == 1 {
|
|
blocked, e := scan(ctx, scanner, dir, []byte(updates[start].Ref))
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
out.ScannedRefs++
|
|
if blocked {
|
|
out.Status = "blocked"
|
|
out.Reason = "ref-name-detection"
|
|
return out
|
|
}
|
|
continue
|
|
}
|
|
if e := getStorage(); e != nil {
|
|
return fail(e)
|
|
}
|
|
var batch []scanInput
|
|
for i := start; i < end; i++ {
|
|
batch = append(batch, scanInput{fmt.Sprintf("ref-%d", i), []byte(updates[i].Ref)})
|
|
}
|
|
blocked, e := scanSmall(ctx, scanner, dir, storage, batch)
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
out.ScannedRefs += len(batch)
|
|
if blocked != "" {
|
|
out.Status = "blocked"
|
|
out.Reason = "ref-name-detection"
|
|
return out
|
|
}
|
|
}
|
|
var pending []scanInput
|
|
flush := func() error {
|
|
if len(pending) == 0 {
|
|
return nil
|
|
}
|
|
if e := getStorage(); e != nil {
|
|
return e
|
|
}
|
|
blocked, e := scanSmall(ctx, scanner, dir, storage, pending)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
out.Scanned += len(pending)
|
|
pending = nil
|
|
if blocked != "" {
|
|
out.Status = "blocked"
|
|
out.BlockedID = blocked
|
|
}
|
|
return nil
|
|
}
|
|
for i, info := range objectMetadata {
|
|
kind, size := info.kind, info.size
|
|
raw, e := git(ctx, nil, int(size)+1, "cat-file", kind, ids[i])
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
if int64(len(raw)) != size {
|
|
return fail(errors.New("object-size-mismatch"))
|
|
}
|
|
if kind == "tree" {
|
|
raw, e = treeContent(raw, width/2)
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
}
|
|
if len(raw) <= directoryInputBytes {
|
|
pending = append(pending, scanInput{ids[i], raw})
|
|
if len(pending) == directoryBatchInputs {
|
|
if e = flush(); e != nil {
|
|
return fail(e)
|
|
}
|
|
if out.Status == "blocked" {
|
|
return out
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
blocked, e := scan(ctx, scanner, dir, raw)
|
|
if e != nil {
|
|
return fail(e)
|
|
}
|
|
out.Scanned++
|
|
if blocked {
|
|
out.Status = "blocked"
|
|
out.BlockedID = ids[i]
|
|
return out
|
|
}
|
|
}
|
|
if err = flush(); err != nil {
|
|
return fail(err)
|
|
}
|
|
if out.Status == "blocked" {
|
|
return out
|
|
}
|
|
after, _, err := refs(ctx)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
if !bytes.Equal(snapshot, after) {
|
|
return fail(errors.New("refs-changed-during-scan"))
|
|
}
|
|
out.Status = "clean"
|
|
return out
|
|
}
|
|
func main() {
|
|
if code, handled := internalProcess(os.Args[1:]); handled {
|
|
os.Exit(code)
|
|
}
|
|
flags := flag.NewFlagSet("receive-secret-scan", flag.ContinueOnError)
|
|
flags.SetOutput(io.Discard)
|
|
binary := flags.String("scanner", "", "reviewed Gitleaks executable")
|
|
out := result{Status: "unverified", Scope: "introduced-reachable-git-objects-and-updated-ref-names", Version: scannerVersion, Reason: "invalid-invocation"}
|
|
if flags.Parse(os.Args[1:]) == nil && *binary != "" && flags.NArg() == 0 {
|
|
raw, err := io.ReadAll(io.LimitReader(os.Stdin, maxInput+1))
|
|
if err == nil {
|
|
updates, e := parseUpdates(raw)
|
|
if e != nil {
|
|
out.Reason = e.Error()
|
|
} else {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
out = check(ctx, updates, *binary)
|
|
cancel()
|
|
}
|
|
}
|
|
}
|
|
// Only fixed reason codes, counts and an object ID may reach the caller.
|
|
encoded, _ := json.Marshal(out)
|
|
fmt.Println(string(encoded))
|
|
if out.Status == "clean" {
|
|
return
|
|
}
|
|
if out.Status == "blocked" {
|
|
os.Exit(1)
|
|
}
|
|
os.Exit(2)
|
|
}
|