Extract Semgrep Python constant-call scanners
ober
f7271551c83210d97ea3f35450adb889931fd8ca
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -489,6 +489,9 @@ Completed in the repo: into `src/semgrep/engine/js-eval-scan.ss` - extracted TypeScript `import = require(...)` and object-type text scanning into `src/semgrep/engine/ts-type-scan.ss` + - extracted Python constant-call text scanners for `and/or` string + constants and loop-`else` numeric guards into + `src/semgrep/engine/py-constant-scan.ss` Validation at this checkpoint: new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-constant-scan.sls @@ -0,0 +1,300 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-constant-scan.ss + +(library (semgrep engine py-constant-scan) + (export + scan-python-and-or-string-call-pattern + scan-python-loop-else-constant-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (semgrep util literals) (semgrep engine regex-scan) + (semgrep result findings)) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + (def (string-find-substring-from s needle start) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i start]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + (def (string-find-last-substring-before source needle end) + (let ([needle-len (string-length needle)]) + (let loop ([i (- end needle-len)]) + (cond + [(< i 0) #f] + [(and (<= (+ i needle-len) (string-length source)) + (string=? (substring source i (+ i needle-len)) needle)) + i] + [else (loop (- i 1))])))) + (def (regex-fold-matches pattern source proc seed) + (let ([rx (re pattern)] [len (string-length source)]) + (let loop ([start 0] [acc seed]) + (if (> start len) + acc + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next (proc match acc))) + acc)))))) + (def (line-start-before source offset) + (let loop ([i (- offset 1)]) + (cond + [(< i 0) 0] + [(char=? (string-ref source i) #\newline) (+ i 1)] + [else (loop (- i 1))]))) + (def (python-quoted-ellipsis-call-function pattern) + (let ([match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*[\"']\\.\\.\\.[\"'][ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") + (string-trim pattern) + 0)]) + (and match (re-match-group match 1)))) + (def (python-and-or-string-constant-assignment? expr) + (let* ([and-index (string-find-substring expr " and ")] + [or-index (and and-index + (string-find-substring-from + expr + " or " + (+ and-index 5)))]) + (and and-index + or-index + (let ([left (string-trim + (substring expr (+ and-index 5) or-index))] + [right (string-trim + (substring + expr + (+ or-index 4) + (string-length expr)))]) + (and (quoted-string? left) (quoted-string? right)))))) + (define python-assignment-line-regex + "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") + (def (python-and-or-string-constant-vars source) + (regex-fold-matches + python-assignment-line-regex + source + (lambda (match acc) + (let ([name (re-match-group match 2)] + [expr (re-match-group match 3)]) + (if (python-and-or-string-constant-assignment? expr) + (cons name acc) + acc))) + '())) + (def (scan-python-and-or-string-call-pattern + rule + path + source + pattern) + (let ([function-name (python-quoted-ellipsis-call-function + pattern)]) + (and function-name + (let ([vars (python-and-or-string-constant-vars source)]) + (and (not (null? vars)) + (apply + append + (map (lambda (name) + (let ([rx (re (string-append "\\b" function-name + "[ \\t]*\\([ \\t]*" name + "[ \\t]*\\)"))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search + rx + source + start)]) + (if match + (let ([next (max (+ (re-match-start + match) + 1) + (re-match-end + match))]) + (loop + next + (cons + (finding-from-match + rule + path + source + match) + acc))) + (reverse acc))))))) + vars))))))) + (def (python-numeric-call-pattern-spec pattern) + (let ([match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*([0-9]+)[ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") + (string-trim pattern) + 0)]) + (and match + (list + (cons 'function (re-match-group match 1)) + (cons 'value (re-match-group match 2)))))) + (def (spaces n) + (let loop ([i 0] [acc '()]) + (if (= i n) + (list->string (reverse acc)) + (loop (+ i 1) (cons #\space acc))))) + (def (line-indent-at-offset source offset) + (let ([start (line-start-before source offset)] + [len (string-length source)]) + (let loop ([i start] [count 0]) + (if (and (< i len) (char=? (string-ref source i) #\space)) + (loop (+ i 1) (+ count 1)) + count)))) + (def (last-python-assignment-before + source + var + value + before-offset) + (let ([pattern (string-append "(^|\\n)([ \\t]*)" var + "[ \\t]*=[ \\t]*" value "\\b")]) + (regex-fold-matches + pattern + (substring source 0 before-offset) + (lambda (match acc) + (let* ([match-start (re-match-start match)] + [line-start (if (and (< match-start + (string-length source)) + (char=? + (string-ref source match-start) + #\newline)) + (+ match-start 1) + match-start)] + [indent-text (re-match-group match 2)]) + (list + (cons 'start line-start) + (cons 'indent (string-length indent-text))))) + #f))) + (def (last-python-loop-before-else + source + else-index + else-indent) + (let ([for-needle (string-append + "\n" + (spaces else-indent) + "for ")] + [while-needle (string-append + "\n" + (spaces else-indent) + "while ")]) + (let ([for-index (string-find-last-substring-before + source + for-needle + else-index)] + [while-index (string-find-last-substring-before + source + while-needle + else-index)]) + (cond + [(and for-index while-index) (max for-index while-index)] + [for-index for-index] + [while-index while-index] + [else #f])))) + (def (loop-body-has-reachable-break? body) + (and (string-find-substring body "\n break") + (not (string-find-substring + body + "\n continue\n break")))) + (def (python-loop-else-constant-call? + source + var + value + call-start) + (let ([assignment (last-python-assignment-before + source + var + value + call-start)]) + (and assignment + (let* ([assign-start (alist-ref/default + assignment + 'start + #f)] + [assign-indent (alist-ref/default + assignment + 'indent + 0)] + [else-indent (max 0 (- assign-indent 4))] + [else-needle (string-append + "\n" + (spaces else-indent) + "else:")] + [else-index (and assign-start + (string-find-last-substring-before + source + else-needle + assign-start))] + [call-indent (line-indent-at-offset + source + call-start)]) + (and else-index + (or (> call-indent else-indent) + (let ([loop-index (last-python-loop-before-else + source + else-index + else-indent)]) + (and loop-index + (not (loop-body-has-reachable-break? + (substring + source + loop-index + else-index))))))))))) + (def (scan-python-loop-else-constant-pattern + rule + path + source + pattern) + (let ([spec (python-numeric-call-pattern-spec pattern)]) + (and spec + (let* ([function-name (alist-ref/default spec 'function "")] + [value (alist-ref/default spec 'value "")] + [rx (re (string-append + "\\b" + function-name + "[ \\t]*\\([ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*\\)"))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (if (null? acc) #f (reverse acc)) + (let ([match (re-search rx source start)]) + (if match + (let* ([var (re-match-group match 1)] + [call-start (re-match-start match)] + [next (max (+ call-start 1) + (re-match-end match))]) + (loop + next + (if (python-loop-else-constant-call? + source + var + value + call-start) + (cons + (finding-from-match + rule + path + source + match) + acc) + acc))) + (if (null? acc) #f (reverse acc))))))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -22,8 +22,10 @@ (semgrep engine js-eval-scan) (semgrep engine js-vardef-scan) (semgrep engine js-decorator-scan) - (semgrep engine markup-scan) (semgrep engine regex-scan) - (semgrep engine rule-plan) (semgrep engine regex-support) + (semgrep engine markup-scan) + (semgrep engine py-constant-scan) + (semgrep engine regex-scan) (semgrep engine rule-plan) + (semgrep engine regex-support) (semgrep engine ts-decorator-scan) (semgrep engine ts-type-scan) (semgrep engine ts-query-scan) (semgrep engine text-support) (semgrep rule parse-rule) @@ -512,12 +514,6 @@ object-end match block-start) acc))) (reverse acc)))))))))) - (def (python-quoted-ellipsis-call-function pattern) - (let ([match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*[\"']\\.\\.\\.[\"'][ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") - (string-trim pattern) - 0)]) - (and match (re-match-group match 1)))) (def (last-char-index-before source ch start end) (let loop ([i (- end 1)]) (cond @@ -725,91 +721,6 @@ next (if finding (cons finding acc) acc))) (if (null? acc) #f (reverse acc)))))))))) - (def (python-and-or-string-constant-assignment? expr) - (let* ([and-index (string-find-substring expr " and ")] - [or-index (and and-index - (string-find-substring-from - expr - " or " - (+ and-index 5)))]) - (and and-index - or-index - (let ([left (string-trim - (substring expr (+ and-index 5) or-index))] - [right (string-trim - (substring - expr - (+ or-index 4) - (string-length expr)))]) - (and (quoted-string? left) (quoted-string? right)))))) - (def python-assignment-line-regex - "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") - (def (python-and-or-string-constant-vars source) - (regex-fold-matches - python-assignment-line-regex - source - (lambda (match acc) - (let ([name (re-match-group match 2)] - [expr (re-match-group match 3)]) - (if (python-and-or-string-constant-assignment? expr) - (cons name acc) - acc))) - '())) - (def (scan-python-and-or-string-call-pattern - rule - path - source - pattern) - (let ([function-name (python-quoted-ellipsis-call-function - pattern)]) - (and function-name - (let ([vars (python-and-or-string-constant-vars source)]) - (and (not (null? vars)) - (apply - append - (map (lambda (name) - (let ([rx (re (string-append "\\b" function-name - "[ \\t]*\\([ \\t]*" name - "[ \\t]*\\)"))] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search - rx - source - start)]) - (if match - (let ([next (max (+ (re-match-start - match) - 1) - (re-match-end - match))]) - (loop - next - (cons - (finding-from-match - rule - path - source - match) - acc))) - (reverse acc))))))) - vars))))))) - (def (python-numeric-call-pattern-spec pattern) - (let ([match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*([0-9]+)[ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") - (string-trim pattern) - 0)]) - (and match - (list - (cons 'function (re-match-group match 1)) - (cons 'value (re-match-group match 2)))))) - (def (spaces n) - (let loop ([i 0] [acc '()]) - (if (= i n) - (list->string (reverse acc)) - (loop (+ i 1) (cons #\space acc))))) (def (line-indent-at-offset source offset) (let ([start (line-start-before source offset)] [len (string-length source)]) @@ -817,143 +728,6 @@ (if (and (< i len) (char=? (string-ref source i) #\space)) (loop (+ i 1) (+ count 1)) count)))) - (def (last-python-assignment-before - source - var - value - before-offset) - (let ([pattern (string-append "(^|\\n)([ \\t]*)" var - "[ \\t]*=[ \\t]*" value "\\b")]) - (regex-fold-matches - pattern - (substring source 0 before-offset) - (lambda (match acc) - (let* ([match-start (re-match-start match)] - [line-start (if (and (< match-start - (string-length source)) - (char=? - (string-ref source match-start) - #\newline)) - (+ match-start 1) - match-start)] - [indent-text (re-match-group match 2)]) - (list - (cons 'start line-start) - (cons 'indent (string-length indent-text))))) - #f))) - (def (last-python-loop-before-else - source - else-index - else-indent) - (let ([for-needle (string-append - "\n" - (spaces else-indent) - "for ")] - [while-needle (string-append - "\n" - (spaces else-indent) - "while ")]) - (let ([for-index (string-find-last-substring-before - source - for-needle - else-index)] - [while-index (string-find-last-substring-before - source - while-needle - else-index)]) - (cond - [(and for-index while-index) (max for-index while-index)] - [for-index for-index] - [while-index while-index] - [else #f])))) - (def (loop-body-has-reachable-break? body) - (and (string-find-substring body "\n break") - (not (string-find-substring - body - "\n continue\n break")))) - (def (python-loop-else-constant-call? - source - var - value - call-start) - (let ([assignment (last-python-assignment-before - source - var - value - call-start)]) - (and assignment - (let* ([assign-start (alist-ref/default - assignment - 'start - #f)] - [assign-indent (alist-ref/default - assignment - 'indent - 0)] - [else-indent (max 0 (- assign-indent 4))] - [else-needle (string-append - "\n" - (spaces else-indent) - "else:")] - [else-index (and assign-start - (string-find-last-substring-before - source - else-needle - assign-start))] - [call-indent (line-indent-at-offset - source - call-start)]) - (and else-index - (or (> call-indent else-indent) - (let ([loop-index (last-python-loop-before-else - source - else-index - else-indent)]) - (and loop-index - (not (loop-body-has-reachable-break? - (substring - source - loop-index - else-index))))))))))) - (def (scan-python-loop-else-constant-pattern - rule - path - source - pattern) - (let ([spec (python-numeric-call-pattern-spec pattern)]) - (and spec - (let* ([function-name (alist-ref/default spec 'function "")] - [value (alist-ref/default spec 'value "")] - [rx (re (string-append - "\\b" - function-name - "[ \\t]*\\([ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*\\)"))] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (if (null? acc) #f (reverse acc)) - (let ([match (re-search rx source start)]) - (if match - (let* ([var (re-match-group match 1)] - [call-start (re-match-start match)] - [next (max (+ call-start 1) - (re-match-end match))]) - (loop - next - (if (python-loop-else-constant-call? - source - var - value - call-start) - (cons - (finding-from-match - rule - path - source - match) - acc) - acc))) - (if (null? acc) #f (reverse acc)))))))))) (def (javascript-member-bar-pattern? pattern) (string-find-substring pattern "$X.bar(")) (def (scan-javascript-imported-bar-call-pattern --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -14,6 +14,9 @@ ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") + ("src/semgrep/engine/py-constant-scan.ss" + . + "285FDB423AD7E28C") ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC") ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") @@ -29,7 +32,7 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "676548090DED3083") + ("src/semgrep/scan.ss" . "8401EF172CD4BF5E") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") ("src/semgrep/engine/js-constructor-scan.ss" . new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/py-constant-scan.ss @@ -0,0 +1,275 @@ +(export + scan-python-and-or-string-call-pattern + scan-python-loop-else-constant-pattern) + +(import (except (jerboa prelude) meta atom?) + (std regex) + (semgrep util literals) + (semgrep engine regex-scan) + (semgrep result findings)) + +(def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + +(def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + +(def (string-find-substring-from s needle start) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i start]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + +(def (string-find-last-substring-before source needle end) + (let ([needle-len (string-length needle)]) + (let loop ([i (- end needle-len)]) + (cond + [(< i 0) #f] + [(and (<= (+ i needle-len) (string-length source)) + (string=? (substring source i (+ i needle-len)) needle)) + i] + [else (loop (- i 1))])))) + +(def (regex-fold-matches pattern source proc seed) + (let ([rx (re pattern)] + [len (string-length source)]) + (let loop ([start 0] [acc seed]) + (if (> start len) + acc + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next (proc match acc))) + acc)))))) + +(def (line-start-before source offset) + (let loop ([i (- offset 1)]) + (cond + [(< i 0) 0] + [(char=? (string-ref source i) #\newline) (+ i 1)] + [else (loop (- i 1))]))) + +(def (python-quoted-ellipsis-call-function pattern) + (let ([match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*[\"']\\.\\.\\.[\"'][ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") + (string-trim pattern) + 0)]) + (and match (re-match-group match 1)))) + +(def (python-and-or-string-constant-assignment? expr) + (let* ([and-index (string-find-substring expr " and ")] + [or-index (and and-index + (string-find-substring-from + expr + " or " + (+ and-index 5)))]) + (and and-index + or-index + (let ([left (string-trim + (substring expr + (+ and-index 5) + or-index))] + [right (string-trim + (substring expr + (+ or-index 4) + (string-length expr)))]) + (and (quoted-string? left) + (quoted-string? right)))))) + +(define python-assignment-line-regex + "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") + +(def (python-and-or-string-constant-vars source) + (regex-fold-matches + python-assignment-line-regex + source + (lambda (match acc) + (let ([name (re-match-group match 2)] + [expr (re-match-group match 3)]) + (if (python-and-or-string-constant-assignment? expr) + (cons name acc) + acc))) + '())) + +(def (scan-python-and-or-string-call-pattern rule path source pattern) + (let ([function-name (python-quoted-ellipsis-call-function pattern)]) + (and function-name + (let ([vars (python-and-or-string-constant-vars source)]) + (and (not (null? vars)) + (apply append + (map (lambda (name) + (let ([rx (re (string-append + "\\b" + function-name + "[ \\t]*\\([ \\t]*" + name + "[ \\t]*\\)"))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start + match) + 1) + (re-match-end + match))]) + (loop next + (cons + (finding-from-match + rule + path + source + match) + acc))) + (reverse acc))))))) + vars))))))) + +(def (python-numeric-call-pattern-spec pattern) + (let ([match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*([0-9]+)[ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") + (string-trim pattern) + 0)]) + (and match + (list (cons 'function (re-match-group match 1)) + (cons 'value (re-match-group match 2)))))) + +(def (spaces n) + (let loop ([i 0] [acc '()]) + (if (= i n) + (list->string (reverse acc)) + (loop (+ i 1) (cons #\space acc))))) + +(def (line-indent-at-offset source offset) + (let ([start (line-start-before source offset)] + [len (string-length source)]) + (let loop ([i start] [count 0]) + (if (and (< i len) + (char=? (string-ref source i) #\space)) + (loop (+ i 1) (+ count 1)) + count)))) + +(def (last-python-assignment-before source var value before-offset) + (let ([pattern (string-append + "(^|\\n)([ \\t]*)" + var + "[ \\t]*=[ \\t]*" + value + "\\b")]) + (regex-fold-matches + pattern + (substring source 0 before-offset) + (lambda (match acc) + (let* ([match-start (re-match-start match)] + [line-start (if (and (< match-start (string-length source)) + (char=? (string-ref source match-start) + #\newline)) + (+ match-start 1) + match-start)] + [indent-text (re-match-group match 2)]) + (list (cons 'start line-start) + (cons 'indent (string-length indent-text))))) + #f))) + +(def (last-python-loop-before-else source else-index else-indent) + (let ([for-needle (string-append "\n" (spaces else-indent) "for ")] + [while-needle (string-append "\n" (spaces else-indent) "while ")]) + (let ([for-index (string-find-last-substring-before + source + for-needle + else-index)] + [while-index (string-find-last-substring-before + source + while-needle + else-index)]) + (cond + [(and for-index while-index) (max for-index while-index)] + [for-index for-index] + [while-index while-index] + [else #f])))) + +(def (loop-body-has-reachable-break? body) + (and (string-find-substring body "\n break") + (not (string-find-substring body "\n continue\n break")))) + +(def (python-loop-else-constant-call? source var value call-start) + (let ([assignment (last-python-assignment-before + source + var + value + call-start)]) + (and assignment + (let* ([assign-start (alist-ref/default assignment 'start #f)] + [assign-indent (alist-ref/default assignment 'indent 0)] + [else-indent (max 0 (- assign-indent 4))] + [else-needle (string-append + "\n" + (spaces else-indent) + "else:")] + [else-index (and assign-start + (string-find-last-substring-before + source + else-needle + assign-start))] + [call-indent (line-indent-at-offset source call-start)]) + (and else-index + (or (> call-indent else-indent) + (let ([loop-index (last-python-loop-before-else + source + else-index + else-indent)]) + (and loop-index + (not (loop-body-has-reachable-break? + (substring source + loop-index + else-index))))))))))) + +(def (scan-python-loop-else-constant-pattern rule path source pattern) + (let ([spec (python-numeric-call-pattern-spec pattern)]) + (and spec + (let* ([function-name (alist-ref/default spec 'function "")] + [value (alist-ref/default spec 'value "")] + [rx (re (string-append + "\\b" + function-name + "[ \\t]*\\([ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*\\)"))] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (if (null? acc) #f (reverse acc)) + (let ([match (re-search rx source start)]) + (if match + (let* ([var (re-match-group match 1)] + [call-start (re-match-start match)] + [next (max (+ call-start 1) + (re-match-end match))]) + (loop next + (if (python-loop-else-constant-call? + source + var + value + call-start) + (cons (finding-from-match + rule + path + source + match) + acc) + acc))) + (if (null? acc) #f (reverse acc)))))))))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -19,6 +19,7 @@ (semgrep engine js-vardef-scan) (semgrep engine js-decorator-scan) (semgrep engine markup-scan) + (semgrep engine py-constant-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) @@ -564,13 +565,6 @@ acc))) (reverse acc)))))))))) -(def (python-quoted-ellipsis-call-function pattern) - (let ([match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*[\"']\\.\\.\\.[\"'][ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") - (string-trim pattern) - 0)]) - (and match (re-match-group match 1)))) - (def (last-char-index-before source ch start end) (let loop ([i (- end 1)]) (cond @@ -744,91 +738,6 @@ (if finding (cons finding acc) acc))) (if (null? acc) #f (reverse acc)))))))))) -(def (python-and-or-string-constant-assignment? expr) - (let* ([and-index (string-find-substring expr " and ")] - [or-index (and and-index - (string-find-substring-from - expr - " or " - (+ and-index 5)))]) - (and and-index - or-index - (let ([left (string-trim - (substring expr - (+ and-index 5) - or-index))] - [right (string-trim - (substring expr - (+ or-index 4) - (string-length expr)))]) - (and (quoted-string? left) - (quoted-string? right)))))) - -(def python-assignment-line-regex - "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") - -(def (python-and-or-string-constant-vars source) - (regex-fold-matches - python-assignment-line-regex - source - (lambda (match acc) - (let ([name (re-match-group match 2)] - [expr (re-match-group match 3)]) - (if (python-and-or-string-constant-assignment? expr) - (cons name acc) - acc))) - '())) - -(def (scan-python-and-or-string-call-pattern rule path source pattern) - (let ([function-name (python-quoted-ellipsis-call-function pattern)]) - (and function-name - (let ([vars (python-and-or-string-constant-vars source)]) - (and (not (null? vars)) - (apply append - (map (lambda (name) - (let ([rx (re (string-append - "\\b" - function-name - "[ \\t]*\\([ \\t]*" - name - "[ \\t]*\\)"))] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search rx source start)]) - (if match - (let ([next (max (+ (re-match-start - match) - 1) - (re-match-end - match))]) - (loop next - (cons - (finding-from-match - rule - path - source - match) - acc))) - (reverse acc))))))) - vars))))))) - -(def (python-numeric-call-pattern-spec pattern) - (let ([match (re-search - (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t\\r\\n]*\\([ \\t\\r\\n]*([0-9]+)[ \\t\\r\\n]*\\)[ \\t\\r\\n]*;?$") - (string-trim pattern) - 0)]) - (and match - (list (cons 'function (re-match-group match 1)) - (cons 'value (re-match-group match 2)))))) - -(def (spaces n) - (let loop ([i 0] [acc '()]) - (if (= i n) - (list->string (reverse acc)) - (loop (+ i 1) (cons #\space acc))))) - (def (line-indent-at-offset source offset) (let ([start (line-start-before source offset)] [len (string-length source)]) @@ -838,115 +747,6 @@ (loop (+ i 1) (+ count 1)) count)))) -(def (last-python-assignment-before source var value before-offset) - (let ([pattern (string-append - "(^|\\n)([ \\t]*)" - var - "[ \\t]*=[ \\t]*" - value - "\\b")])