perf(scanner): precompute rule meta + per-file pattern resolution
ober
78bec33bf8c0e1ecc39c3cf8099ef5e12afce8c5
--- a/gitsafe/scanner.ss +++ b/gitsafe/scanner.ss @@ -408,15 +408,18 @@ (filter-map custom-pattern->secret-pattern (gitsafe-config-custom-patterns config))) - (def (rule-meta config pattern-id) - (let ([id-str (symbol->string pattern-id)]) + (def (build-rule-meta-table config) + (let ([table (make-hash-table)]) (let loop ([items (gitsafe-config-custom-patterns config)]) (cond - [(null? items) #f] - [(let ([id (custom-ref (car items) "id" #f)]) - (and (string? id) (string=? id id-str))) - (car items)] - [else (loop (cdr items))])))) + [(null? items) table] + [else + (let ([id (custom-ref (car items) "id" #f)]) + (when (and (string? id) + (not (string-empty? id)) + (not (hash-key? table (string->symbol id)))) + (hash-put! table (string->symbol id) (car items))) + (loop (cdr items)))])))) (def (rule-secret-group meta) (let ([n (json-number @@ -467,6 +470,20 @@ (any (lambda (p) (safe-regex-search? p file)) paths) (any (lambda (g) (glob-match? g file)) globs)))) + ;; Resolve each pattern's metadata once per file: drop patterns whose path + ;; gate fails for this file and cache the meta + keywords so the per-line + ;; scan no longer re-scans custom-patterns or re-extracts JSON per line. + (def (resolve-patterns-for-file patterns config file meta-table) + (let loop ([pats patterns] [acc '()]) + (if (null? pats) + (reverse acc) + (let* ([pat (car pats)] + [meta (hash-ref meta-table (secret-pattern-id pat) #f)]) + (if (path-pass? file meta) + (loop (cdr pats) + (cons (vector pat meta (rule-keywords meta)) acc)) + (loop (cdr pats) acc)))))) + (def (rule-allowlist-objects meta) (let ([raw (custom-ref-any meta '("allowlists" "allowlist") (vector))]) (cond @@ -564,28 +581,30 @@ (symbol->string pattern-id) ":" (number->string line-number))) - ;; --- Scan a single line against all patterns --- - ;; Returns list of finding structs. - (def (scan-line file line-number line patterns config) + ;; --- Scan a single line against resolved patterns --- + ;; RESOLVED is a list of #(pattern meta keywords) vectors already filtered + ;; by path-pass? for this file; returns list of finding structs. + (def (scan-line/resolved file line-number line resolved config) ;; Skip very long lines (minified/generated code) (if (> (string-length line) 2000) '() - (let loop ([pats patterns] [results '()]) - (if (null? pats) + (let loop ([rs resolved] [results '()]) + (if (null? rs) (reverse results) - (let* ([pat (car pats)] - [pid (secret-pattern-id pat)] - [meta (rule-meta config pid)]) - (if (not (and (path-pass? file meta) - (keywords-pass? line (rule-keywords meta)))) - (loop (cdr pats) results) + (let* ([r (car rs)] + [pat (vector-ref r 0)] + [meta (vector-ref r 1)] + [keywords (vector-ref r 2)] + [pid (secret-pattern-id pat)]) + (if (not (keywords-pass? line keywords)) + (loop (cdr rs) results) (let ([m (re-search (secret-pattern-pregexp pat) line)]) (if (not m) - (loop (cdr pats) results) + (loop (cdr rs) results) (let* ([full-match (re-match-full m)] [matched (extract-match m (rule-secret-group meta))]) (if (not matched) - (loop (cdr pats) results) + (loop (cdr rs) results) ;; A bare 32-hex value is commonly an ordinary MD5 digest. ;; Require a credential-signalling path or nearby key/token ;; label. Broader entropy patterns retain their existing @@ -599,7 +618,7 @@ (in-digest-context? line (re-match-start m)) (in-git-sha-context? line (re-match-start m) matched) (in-dockerfile-from? line)))) - (loop (cdr pats) results) + (loop (cdr rs) results) ;; Run validator and metadata allowlists before building. (let ([valid? (let ([v (secret-pattern-validator pat)]) (if v (v matched) #t))]) @@ -608,7 +627,7 @@ (allowlisted? matched config) (rule-allowlisted? meta file line matched full-match) (line-suppressed? line pid)) - (loop (cdr pats) results) + (loop (cdr rs) results) (let ([f (make-finding pid (secret-pattern-name pat) @@ -623,13 +642,20 @@ pid line-number) (rule-tags meta))]) - (loop (cdr pats) (cons f results)))))))))))))))) + (loop (cdr rs) (cons f results)))))))))))))))) - (def (scan-fragment fragment patterns config) + (def (scan-line file line-number line patterns config) + (scan-line/resolved + file line-number line + (resolve-patterns-for-file patterns config file + (build-rule-meta-table config)) + config)) + + (def (scan-fragment/resolved fragment resolved config) (let* ([file (scan-fragment-file fragment)] [line-number (scan-fragment-line-number fragment)] [content (scan-fragment-content fragment)] - [base (scan-line file line-number content patterns config)] + [base (scan-line/resolved file line-number content resolved config)] [depth (gitsafe-config-max-decode-depth config)]) (if (or (<= depth 0) (scan-fragment-decoded? fragment)) base @@ -640,35 +666,44 @@ '() (apply append (map (lambda (candidate) - (scan-line file line-number candidate patterns config)) + (scan-line/resolved file line-number candidate resolved config)) decoded)))))))) + (def (scan-fragment fragment patterns config) + (let* ([file (scan-fragment-file fragment)] + [resolved (resolve-patterns-for-file patterns config file + (build-rule-meta-table config))]) + (scan-fragment/resolved fragment resolved config))) + ;; --- Scan with explicit pattern list --- - (def (scan-content/patterns file content patterns config) - (let loop ([lines (string-split content #\newline)] - [line-no 1] - [results '()]) - (if (null? lines) - results - (let ([findings - (scan-fragment - (make-scan-fragment file line-no (car lines) 'file #f #f) - patterns - config)]) - (loop (cdr lines) - (+ line-no 1) - (append (reverse findings) results)))))) + (def (scan-content/patterns file content patterns config meta-table) + (let ([resolved (resolve-patterns-for-file patterns config file meta-table)]) + (let loop ([lines (string-split content #\newline)] + [line-no 1] + [results '()]) + (if (null? lines) + results + (let ([findings + (scan-fragment/resolved + (make-scan-fragment file line-no (car lines) 'file #f #f) + resolved + config)]) + (loop (cdr lines) + (+ line-no 1) + (append (reverse findings) results))))))) ;; --- Scan full file content (string), honouring ML-data sniffing --- (def (scan-content file content config) - (let ([base (active-patterns config)]) + (let ([base (active-patterns config)] + [meta-table (build-rule-meta-table config)]) (scan-content/patterns file content (if (and (gitsafe-config-ml-data-detection config) (ml-training-data-content? file content)) (drop-entropy-patterns base) base) - config))) + config + meta-table))) ;; --- Stderr warning helper --- (def (warn-skipped-large path size-mb limit-mb) @@ -698,10 +733,12 @@ ;; the in-repo .gitsafeignore are hostile input: a predictable ;; file:pattern-id:line line would otherwise silence a known finding. (def (filter-ignored-findings findings config) - (filter (lambda (f) - (not (member (finding-fingerprint f) - (gitsafe-config-baseline-fingerprints config)))) - findings)) + (let ([baseline (make-hash-table)]) + (for-each (lambda (fp) (hash-put! baseline fp #t)) + (gitsafe-config-baseline-fingerprints config)) + (filter (lambda (f) + (not (hash-key? baseline (finding-fingerprint f)))) + findings))) (def *git-empty-tree* "4b825dc642cb6eb9a060e54bf8d69288fbee4904") @@ -746,7 +783,7 @@ 'diff #f #f) - patterns + resolved config)) (diff-hunk-lines hunk)))) hunks)))))