Resolve security audit findings
ober
2512fa711d88862061c9da561b74d6626cd2e5f3
--- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install system tools run: | --- a/.github/workflows/security-baseline.yml +++ b/.github/workflows/security-baseline.yml @@ -13,7 +13,7 @@ jobs: baseline: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Required release files run: | --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ EVIDENCE_DIR ?= dist/release-evidence SBOM_DIR ?= dist/sbom REPRO_DIR ?= dist/reproducibility TARGET_EVIDENCE_DIR ?= dist/target-evidence +INSTALL_SMOKE_DIR ?= dist/install-smoke TEMPLATE_DIR := $(HOME)/.git-templates HOOK_DIR := $(TEMPLATE_DIR)/hooks JERBOA_HOME ?= ../jerboa @@ -27,7 +28,7 @@ SCHEME ?= $(JERBOA_HOME)/.chez/bin/scheme FREEBSD_AMD64_CC ?= $(JERBOA_HOME)/support/cross-cc-freebsd-amd64 XC_LIBDIRS = $(CURDIR):$(JERBOA_HOME)/lib:$(JH)/lib -.PHONY: all build binary run test import-check security audit verify sbom reproducibility-report target-evidence release-evidence install clean help ensure-jerboa-tools linux linux-amd64 linux-arm64 freebsd freebsd-amd64 +.PHONY: all build binary run test import-check security audit verify sbom reproducibility-report target-evidence release-evidence install install-smoke clean help ensure-jerboa-tools linux linux-amd64 linux-arm64 freebsd freebsd-amd64 .DEFAULT_GOAL := help all: binary @@ -78,7 +79,17 @@ audit: security target-evidence: scripts/target-evidence.sh @REPO_ROOT=. TARGET_EVIDENCE_DIR="$(TARGET_EVIDENCE_DIR)" sh scripts/target-evidence.sh -verify: security test binary target-evidence +verify: security test binary install-smoke target-evidence + +install-smoke: binary + rm -rf "$(INSTALL_SMOKE_DIR)" + mkdir -p "$(INSTALL_SMOKE_DIR)/bin" "$(INSTALL_SMOKE_DIR)/input" + install -m 0755 "$(BIN)" "$(INSTALL_SMOKE_DIR)/bin/gitsafe" + cp LICENSE "$(INSTALL_SMOKE_DIR)/input/clean.txt" + cmp "$(BIN)" "$(INSTALL_SMOKE_DIR)/bin/gitsafe" + PATH="$(CURDIR)/$(INSTALL_SMOKE_DIR)/bin:/usr/bin:/bin" gitsafe --version > "$(INSTALL_SMOKE_DIR)/version.txt" + PATH="$(CURDIR)/$(INSTALL_SMOKE_DIR)/bin:/usr/bin:/bin" gitsafe scan "$(INSTALL_SMOKE_DIR)/input" --severity low > "$(INSTALL_SMOKE_DIR)/directory-scan.txt" + printf 'installed_artifact_status=match\ninstalled_directory_scan_status=pass\n' > "$(INSTALL_SMOKE_DIR)/status.txt" sbom: ensure-jerboa-tools REPO_ROOT="$(CURDIR)" SBOM_DIR="$(SBOM_DIR)" JERBUILD="$(JERBUILD)" sh scripts/sbom.sh @@ -95,6 +106,7 @@ release-evidence: $(MAKE) security > $(EVIDENCE_DIR)/security-check.txt 2>&1 $(MAKE) test > $(EVIDENCE_DIR)/test.txt 2>&1 $(MAKE) binary > $(EVIDENCE_DIR)/build.txt 2>&1 + $(MAKE) install-smoke > $(EVIDENCE_DIR)/install-smoke.txt 2>&1 $(MAKE) sbom > $(EVIDENCE_DIR)/sbom-log.txt 2>&1 $(MAKE) reproducibility-report > $(EVIDENCE_DIR)/reproducibility-log.txt 2>&1 $(MAKE) target-evidence > $(EVIDENCE_DIR)/target-evidence.log 2>&1 @@ -120,6 +132,8 @@ release-evidence: install: binary mkdir -p "$(BIN_DIR)" install -m 0755 "$(BIN)" "$(BIN_DIR)/gitsafe" + cmp "$(BIN)" "$(BIN_DIR)/gitsafe" + test "$$($(BIN) --version)" = "$$($(BIN_DIR)/gitsafe --version)" mkdir -p "$(HOOK_DIR)" @for hook in pre-commit pre-push; do \ path="$(HOOK_DIR)/$$hook"; \ --- a/gitsafe/main-binary.ss +++ b/gitsafe/main-binary.ss @@ -97,16 +97,16 @@ ;; --- Scanning dispatch --- -(def (run-scan findings format verbose?) +(def (run-scan findings format verbose? (scanned-files #f)) (if (null? findings) (begin (when (or verbose? (not (string=? format "text"))) - (display-findings findings format verbose?)) - (display-summary findings) + (display-findings findings format verbose? scanned-files)) + (display-summary findings scanned-files) (exit 0)) (begin - (display-findings findings format verbose?) - (display-summary findings) + (display-findings findings format verbose? scanned-files) + (display-summary findings scanned-files) (exit 1)))) (def (cmd-pre-commit config format verbose?) @@ -122,13 +122,17 @@ (def (cmd-scan paths config format verbose?) (if (null? paths) (begin (displayln "gitsafe: error: no paths specified") (exit 2)) - (run-scan (scan-files paths config) format verbose?))) + (call-with-values + (lambda () (scan-files-with-count paths config)) + (lambda (findings scanned-files) + (run-scan findings format verbose? scanned-files))))) (def (cmd-stdin config format verbose?) (run-scan (scan-content "<stdin>" (get-string-all (current-input-port)) config) format - verbose?)) + verbose? + 1)) ;; --- Argument parsing (identical to main.ss) --- --- a/gitsafe/main.ss +++ b/gitsafe/main.ss @@ -115,16 +115,16 @@ ;; --- Main scanning dispatch --- -(def (run-scan findings format verbose?) +(def (run-scan findings format verbose? (scanned-files #f)) (if (null? findings) (begin (when (or verbose? (not (string=? format "text"))) - (display-findings findings format verbose?)) - (display-summary findings) + (display-findings findings format verbose? scanned-files)) + (display-summary findings scanned-files) (exit 0)) (begin - (display-findings findings format verbose?) - (display-summary findings) + (display-findings findings format verbose? scanned-files) + (display-summary findings scanned-files) (exit 1)))) (def (cmd-pre-commit config format verbose?) @@ -140,13 +140,17 @@ (def (cmd-scan paths config format verbose?) (if (null? paths) (begin (displayln "gitsafe: error: no paths specified") (exit 2)) - (run-scan (scan-files paths config) format verbose?))) + (call-with-values + (lambda () (scan-files-with-count paths config)) + (lambda (findings scanned-files) + (run-scan findings format verbose? scanned-files))))) (def (cmd-stdin config format verbose?) (run-scan (scan-content "<stdin>" (get-string-all (current-input-port)) config) format - verbose?)) + verbose? + 1)) ;; --- Argument parsing --- --- a/gitsafe/output.ss +++ b/gitsafe/output.ss @@ -112,13 +112,16 @@ (hash-put! ht "tags" (finding-tags f)) ht)) - (def (display-findings-json findings) + (def (display-findings-json findings scanned-files) (let* ([objs (map finding->json-obj findings)] [counts (findings-count-by-severity findings)] - [files (length (unique (map finding-file findings)))] + [files-with-findings + (length (unique (map finding-file findings)))] [summary (let ([s (make-hash-table)]) (hash-put! s "total" (length findings)) - (hash-put! s "files" files) + (hash-put! s "files_with_findings" files-with-findings) + (when (and scanned-files (integer? scanned-files)) + (hash-put! s "files_scanned" scanned-files)) (hash-put! s "critical" (hash-ref counts "critical" 0)) (hash-put! s "high" (hash-ref counts "high" 0)) (hash-put! s "medium" (hash-ref counts "medium" 0)) @@ -220,9 +223,10 @@ ;; --- Summary line --- - (def (display-summary findings) + (def (display-summary findings (scanned-files #f)) (let ([n (length findings)] - [files (length (unique (map finding-file findings)))]) + [files-with-findings + (length (unique (map finding-file findings)))]) (if (= n 0) (begin (when (color-enabled?) @@ -230,7 +234,13 @@ (display "gitsafe: ") (when (color-enabled?) (display "\x1b;[0m")) - (display "no secrets detected.") + (display "no secrets detected") + (when (and scanned-files (integer? scanned-files)) + (display " in ") + (display scanned-files) + (display " scanned file") + (when (not (= scanned-files 1)) (display "s"))) + (display ".") (newline)) (begin (display (severity-color 'critical)) @@ -239,9 +249,15 @@ (display " secret") (when (> n 1) (display "s")) (display " found in ") - (display files) + (display files-with-findings) (display " file") - (when (> files 1) (display "s")) + (when (not (= files-with-findings 1)) (display "s")) + (display " with findings") + (when (and scanned-files (integer? scanned-files)) + (display " out of ") + (display scanned-files) + (display " scanned file") + (when (not (= scanned-files 1)) (display "s"))) (display ". Commit blocked.") (display (color-reset)) (newline) @@ -250,10 +266,10 @@ ;; --- Main dispatch --- - (def (display-findings findings format verbose?) + (def (display-findings findings format verbose? (scanned-files #f)) (match format ["json" - (display-findings-json findings)] + (display-findings-json findings scanned-files)] ["sarif" (display-findings-sarif findings)] [_ --- a/gitsafe/patterns.ss +++ b/gitsafe/patterns.ss @@ -351,6 +351,19 @@ ;; --- MEDIUM patterns --- + ;; A raw 128-bit token is indistinguishable from an MD5 digest without + ;; context. The scanner therefore enables this pattern only for strongly + ;; secret-signalling paths (for example `.repl-token`) or assignment-like + ;; token/key context. + (def pat-raw-hex-bearer + (make-secret-pattern + 'raw-hex-bearer + "Raw 128-bit Hex Bearer Token" + 'high + (re "(?:^|[^0-9A-Fa-f])([0-9A-Fa-f]{32})(?:$|[^0-9A-Fa-f])") + #f + "Raw 32-hex bearer token in a credential path or secret context")) + (def pat-connection-string (make-secret-pattern 'connection-string @@ -409,6 +422,7 @@ pat-jwt pat-basic-auth-url ;; Medium + pat-raw-hex-bearer pat-connection-string pat-high-entropy-hex pat-high-entropy-base64)) --- a/gitsafe/scanner.ss +++ b/gitsafe/scanner.ss @@ -19,6 +19,7 @@ scan-staged scan-push-range scan-files + scan-files-with-count skip-file? ml-training-data-content? finding-fingerprint-value) @@ -225,6 +226,20 @@ (def *url-sensitive-patterns* '(high-entropy-base64 high-entropy-hex)) + (def *secret-bearing-filename-re* + (re "(?i:^(?:\\.?(?:repl|jerboa|jcode|access|auth)[-_]?token|\\.?(?:api|private|auth)[-_]?key|\\.?(?:token|secret|credentials?))(?:\\.(?:txt|key|env|json))?$)")) + + (def *secret-assignment-prefix-re* + (re "(?i:(?:token|secret|credential|api[-_]?key|auth[-_]?key|access[-_]?key)[^0-9A-Fa-f]{0,32}$)")) + + (def (raw-hex-bearer-context? file line match-start) + (or (and (re-search *secret-bearing-filename-re* + (path-strip-directory file)) + #t) + (let* ([start (max 0 (- match-start 96))] + [prefix (substring line start match-start)]) + (and (re-search *secret-assignment-prefix-re* prefix) #t)))) + ;; Returns #t if the position match-start in line follows a "://" sequence, ;; indicating the match is part of a URL path, query string, or fragment. ;; Looks back up to 300 chars to handle long URLs before the match. @@ -570,14 +585,19 @@ [matched (extract-match m (rule-secret-group meta))]) (if (not matched) (loop (cdr pats) results) - ;; For high-noise entropy patterns, suppress matches inside - ;; URLs, OCI content digests, Git object IDs, and Dockerfile - ;; FROM lines. - (if (and (member pid *url-sensitive-patterns*) - (or (in-url-context? line (re-match-start m)) - (in-digest-context? line (re-match-start m)) - (in-git-sha-context? line (re-match-start m) matched) - (in-dockerfile-from? line))) + ;; A bare 32-hex value is commonly an ordinary MD5 digest. + ;; Require a credential-signalling path or nearby key/token + ;; label. Broader entropy patterns retain their existing + ;; URL, digest, Git object, and Docker suppression. + (if (or + (and (eq? pid 'raw-hex-bearer) + (not (raw-hex-bearer-context? + file line (re-match-start m)))) + (and (member pid *url-sensitive-patterns*) + (or (in-url-context? line (re-match-start m)) + (in-digest-context? line (re-match-start m)) + (in-git-sha-context? line (re-match-start m) matched) + (in-dockerfile-from? line)))) (loop (cdr pats) results) ;; Run validator and metadata allowlists before building. (let ([valid? (let ([v (secret-pattern-validator pat)]) @@ -848,7 +868,7 @@ (apply append (map expand-scan-path paths))) ;; --- Top-level: scan specific files or directories --- - (def (scan-files paths config) + (def (scan-files-with-count paths config) (let ([ignore-pats (load-ignorefile)] [limit-mb (gitsafe-config-max-file-size-mb config)]) ;; Phase 1: filter by size sequentially so warnings don't interleave @@ -870,13 +890,20 @@ #f] [else #t])) (expand-scan-paths paths))]) - (filter-ignored-findings - (pmap-files - (lambda (path) - (let ([content (read-file-string path)]) - (scan-content path content config))) - to-scan) - ignore-pats - config)))) + (values + (filter-ignored-findings + (pmap-files + (lambda (path) + (let ([content (read-file-string path)]) + (scan-content path content config))) + to-scan) + ignore-pats + config) + (length to-scan))))) + + (def (scan-files paths config) + (call-with-values + (lambda () (scan-files-with-count paths config)) + (lambda (findings scanned-files) findings))) ) ;; end library --- a/test/test-gitsafe.ss +++ b/test/test-gitsafe.ss @@ -534,6 +534,62 @@ c)]) (check-equal? '() findings))) + (test-case "raw 32-hex bearer is required in a .repl-token file" + (let* ([token "d9f4a7c2e81b6503ac97d42f1e8b36c0"] + [findings (scan-content ".repl-token" token (default-config))] + [raw (filter (lambda (f) + (eq? (finding-pattern-id f) 'raw-hex-bearer)) + findings)]) + (check-predicate raw pair?) + (check-equal? token (finding-matched-text (car raw))))) + + (test-case "raw 32-hex bearer is detected in explicit token context" + (let* ([token "71ac9e04d8b253f6c10e7a49b5328df0"] + [findings (scan-content "service.conf" + (string-append "repl_token = " token "\n") + (default-config))]) + (check-predicate + (filter (lambda (f) + (eq? (finding-pattern-id f) 'raw-hex-bearer)) + findings) + pair?))) + + (test-case "ordinary 32-hex digests remain negative controls" + (let* ([digest "d41d8cd98f00b204e9800998ecf8427e"] + [findings (scan-content "checksums.txt" + (string-append "MD5 (empty) = " digest "\n") + (default-config))]) + (check-equal? '() + (filter (lambda (f) + (eq? (finding-pattern-id f) 'raw-hex-bearer)) + findings)))) + + (test-case "JSON output separates scanned files from files with findings" + (let* ([findings (scan-content ".repl-token" + "d9f4a7c2e81b6503ac97d42f1e8b36c0\n" + (default-config))] + [out (open-output-string)]) + (parameterize ([current-output-port out]) + (display-findings findings "json" #f 9)) + (let ([json (get-output-string out)]) + (check-predicate json + (lambda (s) (string-contains s "\"files_scanned\":9"))) + (check-predicate json + (lambda (s) (string-contains s "\"files_with_findings\":1")))))) + + (test-case "file scan reports the number of files actually scanned" + (call-with-values + (lambda () + (scan-files-with-count + '("test/fixtures/false-positives.txt") + (default-config))) + (lambda (findings scanned-files) + (check-equal? 1 scanned-files) + (check-equal? '() + (filter (lambda (f) + (eq? (finding-pattern-id f) 'raw-hex-bearer)) + findings))))) + (test-case "scan-content: git shas are not flagged as entropy secrets" (let* ([c (default-config)] [findings (scan-content "rebase.log"