Fix critical scanner bugs and add new secret patterns
ober
acdcdeb54b7c5c69c7c4ea5be7cc2fbd561a32c5
--- a/gitsafe/git.ss +++ b/gitsafe/git.ss @@ -40,12 +40,17 @@ ;; --- Internal helpers --- ;; Run a git command and return trimmed stdout. - ;; Returns "" on error (non-zero exit). + ;; Returns "" on error (non-zero exit), but warns on stderr. (def (git-output args) (try (let ([out (run-process args)]) (string-trim out)) - (catch (e) ""))) + (catch (e) + (let ([p (current-error-port)]) + (display "gitsafe: warning: git command failed: " p) + (display (string-join args " ") p) + (newline p)) + ""))) ;; Run git command and return exit code. (def (git-exit args) --- a/gitsafe/patterns.ss +++ b/gitsafe/patterns.ss @@ -59,10 +59,12 @@ (loop (+ i 1) (- e (* p (log p 2)))))))))))))) (def (not-placeholder? str) - ;; Reject common test/example placeholder strings + ;; Reject common test/example placeholder strings. + ;; Only match placeholder words at word boundaries (start/end or separator) + ;; to avoid rejecting real secrets that happen to contain "test" or "fake". (not (or (pregexp-match "^[Xx]+$" str) (pregexp-match "^[Aa]+$" str) - (pregexp-match "(?i:example|placeholder|your[_-]?key|test|fake|dummy|replace)" str) + (pregexp-match "(?i:(?:^|[_.-])(?:example|placeholder|dummy|sample|your[_-]?(?:api[_-]?)?key|replace[_-]?me|change[_-]?me|insert[_-]?here)(?:$|[_.-]))" str) (string=? str "") (< (string-length str) 8)))) @@ -127,10 +129,19 @@ 'anthropic-api-key "Anthropic API Key" 'critical - (pregexp "sk-ant-(?:api03-)?[A-Za-z0-9_-]{90,200}") + (pregexp "sk-ant-(?:[a-z0-9]+-)?[A-Za-z0-9_-]{80,200}") (lambda (m) (has-prefix? m "sk-ant-")) "Anthropic Claude API Key")) + (def pat-openai-svcacct-key + (make-secret-pattern + 'openai-svcacct-key + "OpenAI Service Account Key" + 'critical + (pregexp "sk-svcacct-[A-Za-z0-9_-]{40,200}") + #f + "OpenAI Service Account API Key")) + (def pat-stripe-secret (make-secret-pattern 'stripe-secret @@ -145,10 +156,19 @@ 'private-key-pem "Private Key (PEM)" 'critical - (pregexp "-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----") + (pregexp "-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP |ENCRYPTED )?PRIVATE KEY-----") #f "PEM-encoded private key block")) + (def pat-putty-private-key + (make-secret-pattern + 'putty-private-key + "PuTTY Private Key" + 'critical + (pregexp "PuTTY-User-Key-File-[0-9]+:") + #f + "PuTTY PPK private key file")) + ;; --- HIGH patterns --- (def pat-generic-api-key @@ -264,10 +284,10 @@ 'basic-auth-url "Credentials in URL" 'high - (pregexp "[a-z+]+://([^:@\\s]+):([^:@\\s]+)@[^\\s\"']+") - (lambda (m) (not (or (string=? m "localhost") - (string=? m "user") - (string=? m "username")))) + (pregexp "[a-z+]+://[^:@\\s]+:[^:@\\s]+@[^\\s\"']+") + (lambda (m) (not (pregexp-match + "(?i://(?:user|username|admin|root):(?:pass|password|passwd|secret|changeme|x{3,})@)" + m))) "URL with embedded user:password credentials")) ;; --- MEDIUM patterns --- @@ -310,9 +330,11 @@ pat-github-fine-grained pat-openai-key pat-openai-project-key + pat-openai-svcacct-key pat-anthropic-key pat-stripe-secret pat-private-key-pem + pat-putty-private-key ;; High pat-generic-api-key pat-generic-secret --- a/gitsafe/scanner.ss +++ b/gitsafe/scanner.ss @@ -99,11 +99,18 @@ ;; --- Extract matched substring from pregexp-match-positions result --- (def (extract-match line positions) - ;; positions is ((start . end) ...) — first pair is the full match + ;; positions is ((start . end) ...) — first pair is the full match, + ;; rest are capture groups. Use the last non-#f capture group if any, + ;; so validators receive the meaningful value (e.g. the key itself) + ;; rather than the full match including boundary/context chars. (and (pair? positions) - (let* ([pair (car positions)] - [s (car pair)] - [e (cdr pair)]) + (let* ([best (let loop ([rest (cdr positions)] [last-good #f]) + (cond + [(null? rest) (or last-good (car positions))] + [(car rest) (loop (cdr rest) (car rest))] + [else (loop (cdr rest) last-good)]))] + [s (car best)] + [e (cdr best)]) (substring line s e)))) ;; --- Get active patterns given config --- --- a/test/fixtures/fake-secrets.txt +++ b/test/fixtures/fake-secrets.txt @@ -16,6 +16,9 @@ GH_TOKEN=github_pat_11ABCDEFG0abcdefghij_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklm # OpenAI API key (fake, contains T3BlbkFJ marker) OPENAI_API_KEY=sk-abcdefghijklmnopqrstT3BlbkFJabcdefghijklmnopqrst +# OpenAI Service Account key (fake) +OPENAI_SVCACCT_KEY=sk-svcacct-ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrst + # Anthropic API key (fake, sk-ant- prefix, ~100 chars) ANTHROPIC_API_KEY=sk-ant-api03-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNO --- a/test/test-gitsafe.ss +++ b/test/test-gitsafe.ss @@ -137,6 +137,30 @@ "sk_live_ABCDEFGHIJKLMNOPQRSTUVWXYZabc") (lambda (m) m)))) + (test-case "private-key-pem: detects OpenSSH private key" + (let ([pat (car (filter (lambda (p) (eq? (secret-pattern-id p) 'private-key-pem)) + (all-patterns)))]) + (check-predicate + (pregexp-match (secret-pattern-pregexp pat) + "-----BEGIN OPENSSH PRIVATE KEY-----") + (lambda (m) m)))) + + (test-case "private-key-pem: detects encrypted private key" + (let ([pat (car (filter (lambda (p) (eq? (secret-pattern-id p) 'private-key-pem)) + (all-patterns)))]) + (check-predicate + (pregexp-match (secret-pattern-pregexp pat) + "-----BEGIN ENCRYPTED PRIVATE KEY-----") + (lambda (m) m)))) + + (test-case "putty-private-key: detects PuTTY key file" + (let ([pat (car (filter (lambda (p) (eq? (secret-pattern-id p) 'putty-private-key)) + (all-patterns)))]) + (check-predicate + (pregexp-match (secret-pattern-pregexp pat) + "PuTTY-User-Key-File-3: ssh-rsa") + (lambda (m) m)))) + )) ;; ============================================================ @@ -224,12 +248,56 @@ (check-equal? #f (skip-file? "config.json" c)) (check-equal? #f (skip-file? "Makefile" c)))) + (test-case "extract-match: AWS key with boundary chars extracts key only" + (let* ([c (default-config)] + [patterns (filter (lambda (p) (eq? (secret-pattern-id p) 'aws-access-key)) + (all-patterns))] + [findings (scan-line "test.txt" 1 + "aws_access_key_id = AKIAIOSFODNN7EXAMPLS" + patterns c)]) + (check-predicate findings pair?) + (check-equal? "AKIAIOSFODNN7EXAMPLS" + (finding-matched-text (car findings))))) + + (test-case "extract-match: generic-secret extracts value not whole assignment" + (let* ([c (default-config)] + [patterns (filter (lambda (p) (eq? (secret-pattern-id p) 'generic-secret)) + (all-patterns))] + [findings (scan-line "test.txt" 1 + "secret = \"xK9mP2nQrT5vWy8zA3bCdEfGhJkLmNpQ\"" + patterns c)]) + (check-predicate findings pair?) + (check-equal? "xK9mP2nQrT5vWy8zA3bCdEfGhJkLmNpQ" + (finding-matched-text (car findings))))) + + (test-case "not-placeholder: does not reject keys containing test/fake substrings" + (let* ([c (default-config)] + [patterns (filter (lambda (p) (eq? (secret-pattern-id p) 'generic-secret)) + (all-patterns))] + [findings (scan-line "test.txt" 1 + "secret = \"testX9mP2nQrT5vWy8zA3bCdEfGhJk\"" + patterns c)]) + (check-predicate findings pair?))) + + (test-case "basic-auth-url: rejects placeholder credentials" + (let* ([c (default-config)] + [patterns (filter (lambda (p) (eq? (secret-pattern-id p) 'basic-auth-url)) + (all-patterns))] + [real (scan-line "test.txt" 1 + "DATABASE_URL=postgres://admin:s3cr3tP4ss@db.example.com:5432/mydb" + patterns c)] + [fake (scan-line "test.txt" 1 + "DATABASE_URL=postgres://user:password@localhost:5432/mydb" + patterns c)]) + (check-predicate real pair?) + (check-equal? '() fake))) + (test-case "scan-content: detects secrets in fixture file" (let* ([c (default-config)] [content (read-file-string "test/fixtures/fake-secrets.txt")] [findings (scan-content "fake-secrets.txt" content c)]) (check-predicate findings pair?) - (check-predicate (length findings) (lambda (n) (>= n 5))))) + (check-predicate (length findings) (lambda (n) (>= n 10))))) (test-case "scan-content: jcode.json fixture is caught" (let* ([c (default-config)]