Extract Semgrep Python string scanner
ober
6f5e3905bf4527223395c5f0a9eb2fa767f53f5e
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -492,6 +492,8 @@ Completed in the repo: - extracted Python constant-call text scanners for `and/or` string constants and loop-`else` numeric guards into `src/semgrep/engine/py-constant-scan.ss` + - extracted Python string-call pattern parsing and scan dispatch into + `src/semgrep/engine/py-string-scan.ss` Validation at this checkpoint: new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-string-scan.sls @@ -0,0 +1,150 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-string-scan.ss + +(library (semgrep engine py-string-scan) + (export scan-python-string-call-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) + (std regex) + (semgrep engine text-support)) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (last-char-index-before source ch start end) + (let loop ([i (- end 1)]) + (cond + [(< i start) #f] + [(char=? (string-ref source i) ch) i] + [else (loop (- i 1))]))) + (def (skip-pattern-horizontal source i end) + (let loop ([j i]) + (if (and (< j end) + (let ([ch (string-ref source j)]) + (or (char=? ch #\space) + (char=? ch #\tab) + (char=? ch #\return) + (char=? ch #\newline)))) + (loop (+ j 1)) + j))) + (def (python-call-string-pattern-spec pattern) + (let* ([trimmed (string-trim pattern)] + [match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\(") + trimmed + 0)]) + (and match + (let* ([function-name (re-match-group match 1)] + [open (- (re-match-end match) 1)] + [len (string-length trimmed)] + [arg-start0 (skip-pattern-horizontal + trimmed + (+ open 1) + len)] + [f-prefix? (and (< arg-start0 len) + (let ([ch (string-ref + trimmed + arg-start0)]) + (or (char=? ch #\f) + (char=? ch #\F))))] + [arg-start (if f-prefix? + (+ arg-start0 1) + arg-start0)]) + (and (< arg-start len) + (let ([quote (string-ref trimmed arg-start)]) + (and (or (char=? quote #\") (char=? quote #\')) + (let* ([close-paren (last-char-index-before + trimmed + #\) + (+ arg-start 1) + len)] + [quote-end (and close-paren + (last-char-index-before + trimmed + quote + (+ arg-start 1) + close-paren))] + [tail-start (and quote-end + close-paren + (skip-pattern-horizontal + trimmed + (+ quote-end 1) + close-paren))]) + (and close-paren + quote-end + (= tail-start close-paren) + (let ([content (substring + trimmed + (+ arg-start 1) + quote-end)]) + (list + (cons "function" function-name) + (cons "expected" content) + (cons + "wildcard-string" + (string=? + content + "..."))))))))))))) + (def (scan-python-string-call-pattern rule path source + pattern constant-string-value) + (let ([spec (python-call-string-pattern-spec pattern)]) + (and spec + (let* ([function-name (alist-ref/default + spec + "function" + #f)] + [expected (alist-ref/default spec "expected" #f)] + [ellipsis-match (alist-ref/default + spec + "wildcard-string" + #f)] + [rx (re (string-append + "\\b" + function-name + "[ \\t]*\\("))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (if (null? acc) #f (reverse acc)) + (let ([match (re-search rx source start)]) + (if match + (let* ([call-start (re-match-start match)] + [open (- (re-match-end match) 1)] + [line-end (line-end-after source open)] + [close (last-char-index-before + source + #\) + (+ open 1) + line-end)] + [arg-text (and close + (substring + source + (+ open 1) + close))] + [value (and arg-text + (constant-string-value + arg-text + source + call-start))] + [matches? (and value + (or ellipsis-match + (string=? + value + expected)))] + [finding (and matches? + (finding-for-range-with-bindings rule path source + call-start (+ close 1) + '()))] + [next (if close + (max (+ call-start 1) + (+ close 1)) + (+ (re-match-start match) + 1))]) + (loop + next + (if finding (cons finding acc) acc))) + (if (null? acc) #f (reverse acc))))))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -24,8 +24,8 @@ (semgrep engine js-decorator-scan) (semgrep engine markup-scan) (semgrep engine py-constant-scan) - (semgrep engine regex-scan) (semgrep engine rule-plan) - (semgrep engine regex-support) + (semgrep engine py-string-scan) (semgrep engine regex-scan) + (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) (semgrep engine ts-type-scan) (semgrep engine ts-query-scan) (semgrep engine text-support) (semgrep rule parse-rule) @@ -530,64 +530,6 @@ (char=? ch #\newline)))) (loop (+ j 1)) j))) - (def (python-call-string-pattern-spec pattern) - (let* ([trimmed (string-trim pattern)] - [match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\(") - trimmed - 0)]) - (and match - (let* ([function-name (re-match-group match 1)] - [open (- (re-match-end match) 1)] - [len (string-length trimmed)] - [arg-start0 (skip-pattern-horizontal - trimmed - (+ open 1) - len)] - [f-prefix? (and (< arg-start0 len) - (let ([ch (string-ref - trimmed - arg-start0)]) - (or (char=? ch #\f) - (char=? ch #\F))))] - [arg-start (if f-prefix? - (+ arg-start0 1) - arg-start0)]) - (and (< arg-start len) - (let ([quote (string-ref trimmed arg-start)]) - (and (or (char=? quote #\") (char=? quote #\')) - (let* ([close-paren (last-char-index-before - trimmed - #\) - (+ arg-start 1) - len)] - [quote-end (and close-paren - (last-char-index-before - trimmed - quote - (+ arg-start 1) - close-paren))] - [tail-start (and quote-end - close-paren - (skip-pattern-horizontal - trimmed - (+ quote-end 1) - close-paren))]) - (and close-paren - quote-end - (= tail-start close-paren) - (let ([content (substring - trimmed - (+ arg-start 1) - quote-end)]) - (list - (cons "function" function-name) - (cons "expected" content) - (cons - "wildcard-string" - (string=? - content - "..."))))))))))))) (def (python-fstring-literal? text) (let* ([trimmed (string-trim text)] [len (string-length trimmed)]) @@ -662,65 +604,8 @@ path source pattern) - (let ([spec (python-call-string-pattern-spec pattern)]) - (and spec - (let* ([function-name (alist-ref/default - spec - "function" - #f)] - [expected (alist-ref/default spec "expected" #f)] - [ellipsis-match (alist-ref/default - spec - "wildcard-string" - #f)] - [rx (re (string-append - "\\b" - function-name - "[ \\t]*\\("))] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (if (null? acc) #f (reverse acc)) - (let ([match (re-search rx source start)]) - (if match - (let* ([call-start (re-match-start match)] - [open (- (re-match-end match) 1)] - [line-end (line-end-after source open)] - [close (last-char-index-before - source - #\) - (+ open 1) - line-end)] - [arg-text (and close - (substring - source - (+ open 1) - close))] - [value (and arg-text - (python-string-literal? - arg-text) - (python-constant-string-literal-value - arg-text - source - call-start))] - [matches? (and value - (or ellipsis-match - (string=? - value - expected)))] - [finding (and matches? - (finding-for-range-with-bindings rule path source - call-start (+ close 1) - '()))] - [next (if close - (max (+ call-start 1) - (+ close 1)) - (+ (re-match-start match) - 1))]) - (loop - next - (if finding (cons finding acc) acc))) - (if (null? acc) #f (reverse acc)))))))))) + (scan-python-string-call-pattern rule path source pattern + python-constant-string-literal-value)) (def (line-indent-at-offset source offset) (let ([start (line-start-before source offset)] [len (string-length source)]) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -1,4 +1,7 @@ -(("src/semgrep/util/literals.ss" . "8A094085551B216E") +(("src/semgrep/engine/py-string-scan.ss" + . + "DBAD99DB5C22F6A1") + ("src/semgrep/util/literals.ss" . "8A094085551B216E") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") @@ -32,7 +35,7 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "8401EF172CD4BF5E") + ("src/semgrep/scan.ss" . "912D7E4F9F9BA531") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") ("src/semgrep/engine/js-constructor-scan.ss" . new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/py-string-scan.ss @@ -0,0 +1,130 @@ +(export + scan-python-string-call-pattern) + +(import (except (jerboa prelude) meta atom?) + (std regex) + (semgrep engine text-support)) + +(def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + +(def (last-char-index-before source ch start end) + (let loop ([i (- end 1)]) + (cond + [(< i start) #f] + [(char=? (string-ref source i) ch) i] + [else (loop (- i 1))]))) + +(def (skip-pattern-horizontal source i end) + (let loop ([j i]) + (if (and (< j end) + (let ([ch (string-ref source j)]) + (or (char=? ch #\space) + (char=? ch #\tab) + (char=? ch #\return) + (char=? ch #\newline)))) + (loop (+ j 1)) + j))) + +(def (python-call-string-pattern-spec pattern) + (let* ([trimmed (string-trim pattern)] + [match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\(") + trimmed + 0)]) + (and match + (let* ([function-name (re-match-group match 1)] + [open (- (re-match-end match) 1)] + [len (string-length trimmed)] + [arg-start0 (skip-pattern-horizontal trimmed (+ open 1) len)] + [f-prefix? (and (< arg-start0 len) + (let ([ch (string-ref trimmed arg-start0)]) + (or (char=? ch #\f) + (char=? ch #\F))))] + [arg-start (if f-prefix? (+ arg-start0 1) arg-start0)]) + (and (< arg-start len) + (let ([quote (string-ref trimmed arg-start)]) + (and (or (char=? quote #\") (char=? quote #\')) + (let* ([close-paren (last-char-index-before + trimmed + #\) + (+ arg-start 1) + len)] + [quote-end (and close-paren + (last-char-index-before + trimmed + quote + (+ arg-start 1) + close-paren))] + [tail-start (and quote-end + close-paren + (skip-pattern-horizontal + trimmed + (+ quote-end 1) + close-paren))]) + (and close-paren + quote-end + (= tail-start close-paren) + (let ([content (substring trimmed + (+ arg-start 1) + quote-end)]) + (list (cons "function" function-name) + (cons "expected" content) + (cons "wildcard-string" + (string=? content "..."))))))))))))) + +(def (scan-python-string-call-pattern + rule + path + source + pattern + constant-string-value) + (let ([spec (python-call-string-pattern-spec pattern)]) + (and spec + (let* ([function-name (alist-ref/default spec "function" #f)] + [expected (alist-ref/default spec "expected" #f)] + [ellipsis-match (alist-ref/default spec "wildcard-string" #f)] + [rx (re (string-append "\\b" + function-name + "[ \\t]*\\("))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (if (null? acc) #f (reverse acc)) + (let ([match (re-search rx source start)]) + (if match + (let* ([call-start (re-match-start match)] + [open (- (re-match-end match) 1)] + [line-end (line-end-after source open)] + [close (last-char-index-before + source + #\) + (+ open 1) + line-end)] + [arg-text (and close + (substring source + (+ open 1) + close))] + [value (and arg-text + (constant-string-value + arg-text + source + call-start))] + [matches? (and value + (or ellipsis-match + (string=? value expected)))] + [finding (and matches? + (finding-for-range-with-bindings + rule + path + source + call-start + (+ close 1) + '()))] + [next (if close + (max (+ call-start 1) (+ close 1)) + (+ (re-match-start match) 1))]) + (loop next + (if finding (cons finding acc) acc))) + (if (null? acc) #f (reverse acc)))))))))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -20,6 +20,7 @@ (semgrep engine js-decorator-scan) (semgrep engine markup-scan) (semgrep engine py-constant-scan) + (semgrep engine py-string-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) @@ -583,53 +584,6 @@ (loop (+ j 1)) j))) -(def (python-call-string-pattern-spec pattern) - (let* ([trimmed (string-trim pattern)] - [match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\(") - trimmed - 0)]) - (and match - (let* ([function-name (re-match-group match 1)] - [open (- (re-match-end match) 1)] - [len (string-length trimmed)] - [arg-start0 (skip-pattern-horizontal trimmed (+ open 1) len)] - [f-prefix? (and (< arg-start0 len) - (let ([ch (string-ref trimmed arg-start0)]) - (or (char=? ch #\f) - (char=? ch #\F))))] - [arg-start (if f-prefix? (+ arg-start0 1) arg-start0)]) - (and (< arg-start len) - (let ([quote (string-ref trimmed arg-start)]) - (and (or (char=? quote #\") (char=? quote #\')) - (let* ([close-paren (last-char-index-before - trimmed - #\) - (+ arg-start 1) - len)] - [quote-end (and close-paren - (last-char-index-before - trimmed - quote - (+ arg-start 1) - close-paren))] - [tail-start (and quote-end - close-paren - (skip-pattern-horizontal - trimmed - (+ quote-end 1) - close-paren))]) - (and close-paren - quote-end - (= tail-start close-paren) - (let ([content (substring trimmed - (+ arg-start 1) - quote-end)]) - (list (cons "function" function-name) - (cons "expected" content) - (cons "wildcard-string" - (string=? content "..."))))))))))))) - (def (python-fstring-literal? text) (let* ([trimmed (string-trim text)] [len (string-length trimmed)]) @@ -688,55 +642,12 @@ [else #f]))) (def (scan-python-fstring-call-pattern rule path source pattern) - (let ([spec (python-call-string-pattern-spec pattern)]) - (and spec - (let* ([function-name (alist-ref/default spec "function" #f)] - [expected (alist-ref/default spec "expected" #f)] - [ellipsis-match (alist-ref/default spec "wildcard-string" #f)] - [rx (re (string-append "\\b" - function-name - "[ \\t]*\\("))] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (if (null? acc) #f (reverse acc)) - (let ([match (re-search rx source start)]) - (if match - (let* ([call-start (re-match-start match)] - [open (- (re-match-end match) 1)] - [line-end (line-end-after source open)] - [close (last-char-index-before - source - #\) - (+ open 1) - line-end)] - [arg-text (and close - (substring source - (+ open 1) - close))] - [value (and arg-text - (python-string-literal? arg-text) - (python-constant-string-literal-value - arg-text - source - call-start))] - [matches? (and value - (or ellipsis-match - (string=? value expected)))] - [finding (and matches? - (finding-for-range-with-bindings - rule - path - source - call-start - (+ close 1) - '()))] - [next (if close - (max (+ call-start 1) (+ close 1)) - (+ (re-match-start match) 1))]) - (loop next - (if finding (cons finding acc) acc))) - (if (null? acc) #f (reverse acc)))))))))) + (scan-python-string-call-pattern + rule + path + source + pattern + python-constant-string-literal-value)) (def (line-indent-at-offset source offset) (let ([start (line-start-before source offset)]