fix: filter sequential charsets and CamelCase ident lists from entropy patterns
ober
34b75256b6910eb812dffe4c44c94800f31d4514
--- a/gitsafe/patterns.ss +++ b/gitsafe/patterns.ss @@ -58,6 +58,61 @@ (let ([p (/ (inexact c) n)]) (loop (+ i 1) (- e (* p (log p 2)))))))))))))) + (def (has-sequential-run? str min-run) + ;; Detect a monotonic ascending/descending run of codepoints of + ;; length >= MIN-RUN — catches alphabet/digit charset literals like + ;; "abcdefghij..." or "0123456789". + (let ([len (string-length str)]) + (if (< len min-run) + #f + (let loop ([i 1] [run 1] [dir 0]) + (cond + [(>= run min-run) #t] + [(>= i len) #f] + [else + (let* ([prev (char->integer (string-ref str (- i 1)))] + [cur (char->integer (string-ref str i))] + [delta (- cur prev)]) + (cond + [(and (= delta 1) (or (= dir 0) (= dir 1))) + (loop (+ i 1) (+ run 1) 1)] + [(and (= delta -1) (or (= dir 0) (= dir -1))) + (loop (+ i 1) (+ run 1) -1)] + [else + (loop (+ i 1) 1 0)]))]))))) + + (def (looks-like-identifier-list? str) + ;; Heuristic: a string containing zero digits but >= 4 letter + ;; case-flips between adjacent positions is almost certainly a list + ;; of CamelCase identifiers (e.g. "SearxEngineCaptcha/AccessDenied"), + ;; not a real high-entropy secret. + (let ([len (string-length str)]) + (cond + [(< len 8) #f] + [else + (let loop ([i 0] [flips 0]) + (cond + [(>= i len) (>= flips 4)] + [else + (let ([ch (string-ref str i)]) + (cond + [(char-numeric? ch) #f] + [(< i 1) (loop (+ i 1) flips)] + [else + (let ([prev (string-ref str (- i 1))]) + (cond + [(or (and (char-lower-case? prev) (char-upper-case? ch)) + (and (char-upper-case? prev) (char-lower-case? ch))) + (loop (+ i 1) (+ flips 1))] + [else (loop (+ i 1) flips)]))]))]))]))) + + (def (looks-like-secret? str threshold) + ;; A high-entropy match is only suspicious if it isn't a sequential + ;; charset literal AND isn't a digit-less CamelCase identifier list. + (and (entropy-above? str threshold) + (not (has-sequential-run? str 6)) + (not (looks-like-identifier-list? str)))) + (def *placeholder-re* (re "(?i:(?:^|[_.-])(?:example|placeholder|dummy|sample|your[_-]?(?:api[_-]?)?key|replace[_-]?me|change[_-]?me|insert[_-]?here)(?:$|[_.-]))")) @@ -311,7 +366,7 @@ "High-Entropy Hex String" 'medium (re "[0-9a-f]{40,}") - (lambda (m) (entropy-above? m 3.0)) + (lambda (m) (looks-like-secret? m 3.0)) "Long hex string with high entropy (possible API key or token)")) (def pat-high-entropy-base64 @@ -320,7 +375,7 @@ "High-Entropy Base64 String" 'medium (re "[A-Za-z0-9+/]{40,}={0,2}") - (lambda (m) (entropy-above? m 4.0)) + (lambda (m) (looks-like-secret? m 4.0)) "Long base64 string with high entropy (possible encoded secret)")) ;; --- Pattern registry ---