Extract Semgrep Python constant-propagation core
ober
092f942e4b5f09140574f49f9b10b33aa52a9912
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -496,6 +496,8 @@ Completed in the repo: `src/semgrep/engine/py-string-scan.ss` - extracted reusable Python string/f-string literal evaluation helpers into `src/semgrep/engine/py-string-eval.ss` + - extracted Python constant-propagation evaluation and flow-analysis core + into `src/semgrep/engine/py-constant-prop.ss` - extracted shared comparison evaluation, constant-binding resolution, and metavariable-comparison predicate handling into `src/semgrep/engine/comparison.ss` --- a/lib/semgrep/engine/comparison.sls +++ b/lib/semgrep/engine/comparison.sls @@ -5,7 +5,8 @@ (library (semgrep engine comparison) (export comparison-missing? comparison-value->string constant-bindings-before comparison-value - metavariable-comparison-satisfied?) + metavariable-comparison-satisfied? skip-whitespace + substring-trim outer-pair? find-top-level-binary-operator) (import (except (chezscheme) make-hash-table hash-table? sort sort! printf fprintf format path-extension path-absolute? new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-constant-prop.sls @@ -0,0 +1,934 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-constant-prop.ss + +(library (semgrep engine py-constant-prop) + (export python-cp-missing-value python-cp-missing? + python-cp-any-string? python-cp-stringish? + python-cp-expression-value python-cp-bindings-before + python-cp-set-binding python-cp-flow-binding-ref + python-cp-remove-binding python-cp-flow-expression-value + python-cp-next-line-start python-cp-flow-env-before) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (semgrep engine comparison) (semgrep engine py-string-eval) + (semgrep engine text-support) (semgrep util literals)) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (any? pred xs) + (and (not (null? xs)) + (or (pred (car xs)) (any? pred (cdr xs))))) + (def (sg-filter pred xs) + (let loop ([remaining xs] [acc '()]) + (cond + [(null? remaining) (reverse acc)] + [(pred (car remaining)) + (loop (cdr remaining) (cons (car remaining) acc))] + [else (loop (cdr remaining) acc)]))) + (def (sg-string-prefix? prefix s) + (let ([prefix-len (string-length prefix)] + [len (string-length s)]) + (and (<= prefix-len len) + (string=? (substring s 0 prefix-len) prefix)))) + (def (join-strings xs) + (let loop ([remaining xs] [acc ""]) + (if (null? remaining) + acc + (loop + (cdr remaining) + (string-append acc (car remaining)))))) + (def (code-string-quote? ch) + (or (char=? ch #\") (char=? ch #\') (char=? ch #\`))) + (def (parse-code-string-literal-loop source len quote + content-start i escaped?) + (cond + [(>= i len) #f] + [escaped? + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #f)] + [(char=? (string-ref source i) #\\) + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #t)] + [(and (char=? quote #\`) + (< (+ i 1) len) + (char=? (string-ref source i) #\$) + (char=? (string-ref source (+ i 1)) #\{)) + #f] + [(char=? (string-ref source i) quote) + (cons (substring source content-start i) (+ i 1))] + [else + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #f)])) + (def (parse-code-string-literal-at source start) + (let ([len (string-length source)]) + (if (< start len) + (let ([quote (string-ref source start)]) + (and (code-string-quote? quote) + (parse-code-string-literal-loop source len quote + (+ start 1) (+ start 1) #f))) + #f))) + (def (string-find-substring s needle) + (let ([needle-len (string-length needle)] + [len (string-length s)]) + (let loop ([i 0]) + (cond + [(> (+ i needle-len) len) #f] + [(string=? (substring s i (+ i needle-len)) needle) i] + [else (loop (+ i 1))])))) + (def (identifier-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$))) + (def (line-indent-at-offset source offset) + (let ([start (let loop ([i (- offset 1)]) + (cond + [(< i 0) 0] + [(char=? (string-ref source i) #\newline) (+ i 1)] + [else (loop (- i 1))]))] + [len (string-length source)]) + (let loop ([i start] [count 0]) + (if (and (< i len) (char=? (string-ref source i) #\space)) + (loop (+ i 1) (+ count 1)) + count)))) + (def (python-simple-identifier? text) + (let ([len (string-length text)]) + (and (> len 0) + (let ([first (string-ref text 0)]) + (or (char-alphabetic? first) (char=? first #\_))) + (let loop ([i 1]) + (cond + [(= i len) #t] + [(identifier-char? (string-ref text i)) (loop (+ i 1))] + [else #f]))))) + (def (python-annotated-lhs-name line) + (let ([equals (char-index-from + line + #\= + 0 + (string-length line))]) + (and equals + (let* ([lhs (string-trim (substring line 0 equals))] + [colon (char-index-from + lhs + #\: + 0 + (string-length lhs))] + [name (string-trim + (if colon (substring lhs 0 colon) lhs))]) + (and (python-simple-identifier? name) name))))) + (def (python-block-end + source + header-start + include-if-chain?) + (let* ([header-indent (line-indent-at-offset + source + header-start)] + [header-end (line-end-after source header-start)] + [len (string-length source)]) + (let loop ([line-start (if (< header-end len) + (+ header-end 1) + (+ len 1))] + [last-end header-end]) + (if (> line-start len) + last-end + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [blank? (= first line-end)] + [trimmed (and (not blank?) + (substring source first line-end))] + [comment? (and trimmed + (sg-string-prefix? "#" trimmed))] + [indent (- first line-start)] + [chain-line? (and include-if-chain? + (= indent header-indent) + trimmed + (or (sg-string-prefix? + "elif " + trimmed) + (sg-string-prefix? + "else:" + trimmed)))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (cond + [(or blank? comment?) (loop next last-end)] + [(or (> indent header-indent) chain-line?) + (loop next line-end)] + [else last-end])))))) + (def (python-cp-try-clause-kind line) + (let ([trimmed (string-trim line)]) + (cond + [(string=? trimmed "try:") 'try] + [(or (sg-string-prefix? "except " trimmed) + (sg-string-prefix? "except:" trimmed)) + 'except] + [(string=? trimmed "else:") 'else] + [(string=? trimmed "finally:") 'finally] + [else #f]))) + (def (python-try-statement-end source try-start) + (let* ([try-indent (line-indent-at-offset source try-start)] + [try-line-end (line-end-after source try-start)] + [len (string-length source)]) + (let loop ([current (if (< try-line-end len) + (+ try-line-end 1) + (+ len 1))] + [last-end try-line-end]) + (if (> current len) + last-end + (let* ([line-end (line-end-after source current)] + [first (line-first-nonspace source current line-end)] + [blank? (= first line-end)] + [trimmed (and (not blank?) + (substring source first line-end))] + [comment? (and trimmed + (sg-string-prefix? "#" trimmed))] + [indent (- first current)] + [same-indent-clause? (and (= indent try-indent) + trimmed + (let ([kind (python-cp-try-clause-kind + trimmed)]) + (or (eq? kind 'except) + (eq? kind 'else) + (eq? kind + 'finally))))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (cond + [(or blank? comment?) (loop next last-end)] + [(or (> indent try-indent) same-indent-clause?) + (loop next line-end)] + [else last-end])))))) + (def (python-line-assignment-info + source + line-start + line-end) + (let* ([first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [equals (char-index-from line #\= 0 (string-length line))] + [name (and equals (python-annotated-lhs-name line))]) + (and name + (let* ([name-rel (let ([name-len (string-length name)] + [line-len (string-length line)]) + (let loop ([i 0]) + (cond + [(> (+ i name-len) line-len) #f] + [(string=? + (substring line i (+ i name-len)) + name) + i] + [else (loop (+ i 1))])))] + [rhs (string-trim + (substring + line + (+ equals 1) + (string-length line)))]) + (and name-rel + (list (cons 'name name) + (cons 'name-start (+ first name-rel)) + (cons + 'name-end + (+ first name-rel (string-length name))) + (cons 'rhs rhs) (cons 'first first) + (cons 'line-end line-end))))))) + (def (python-fstring-expression-string + expr + source + before-offset) + (let ([trimmed (string-trim expr)]) + (cond + [(string-find-substring trimmed ":") #f] + [(python-fstring-literal? trimmed) + (python-constant-string-literal-value + trimmed + source + before-offset)] + [else + (let* ([constants (constant-bindings-before + source + before-offset + #f)] + [value (comparison-value trimmed constants #f #f)]) + (and (not (comparison-missing? value)) + (comparison-value->string value)))]))) + (def (python-constant-string-literal-value + text + source + before-offset) + (python-constant-string-literal-value* + text + source + before-offset + python-fstring-expression-string)) + (define python-cp-missing-value + (list 'python-cp-missing-value)) + (def (python-cp-missing? value) + (eq? value python-cp-missing-value)) + (def (python-cp-binding-ref bindings name) + (let ([found (assoc name bindings)]) + (if found (cdr found) python-cp-missing-value))) + (def (python-cp-remove-binding bindings name) + (let loop ([xs bindings]) + (cond + [(null? xs) '()] + [(string=? (caar xs) name) (loop (cdr xs))] + [else (cons (car xs) (loop (cdr xs)))]))) + (def (python-cp-repeat-string text count) + (if (and (number? count) (<= 0 count 256)) + (let loop ([i count] [acc '()]) + (if (= i 0) + (join-strings acc) + (loop (- i 1) (cons text acc)))) + text)) + (def (python-cp-adjacent-string-value expr) + (let ([len (string-length expr)]) + (let loop ([i 0] [parts '()]) + (let ([start (skip-whitespace expr i)]) + (cond + [(>= start len) + (if (null? (cdr parts)) + python-cp-missing-value + (join-strings (reverse parts)))] + [(parse-code-string-literal-at expr start) => + (lambda (literal) + (loop (cdr literal) (cons (car literal) parts)))] + [else python-cp-missing-value]))))) + (def (python-cp-simple-identifier-text? text) + (python-simple-identifier? (string-trim text))) + (def (python-cp-expression-value + expr + bindings + source + before-offset) + (let ([trimmed (string-trim expr)]) + (cond + [(string=? trimmed "") python-cp-missing-value] + [(outer-pair? trimmed #\( #\)) + (let ([inner (substring + trimmed + 1 + (- (string-length trimmed) 1))]) + (let ([adjacent (python-cp-adjacent-string-value inner)]) + (if (python-cp-missing? adjacent) + (python-cp-expression-value + inner + bindings + source + before-offset) + adjacent)))] + [(python-string-literal? trimmed) + (or (python-constant-string-literal-value + trimmed + source + before-offset) + python-cp-missing-value)] + [(or (string=? trimmed "True") (string=? trimmed "true")) + #t] + [(or (string=? trimmed "False") (string=? trimmed "false")) + #f] + [(parse-number-literal trimmed #f) => values] + [(find-top-level-binary-operator trimmed '("+")) => + (lambda (match) + (let* ([index (cdr match)] + [left (python-cp-expression-value + (substring-trim trimmed 0 index) + bindings + source + before-offset)] + [right (python-cp-expression-value + (substring-trim + trimmed + (+ index 1) + (string-length trimmed)) + bindings + source + before-offset)]) + (cond + [(or (python-cp-missing? left) + (python-cp-missing? right)) + python-cp-missing-value] + [(and (string? left) (string? right)) + (string-append left right)] + [(and (number? left) (number? right)) (+ left right)] + [else python-cp-missing-value])))] + [(find-top-level-binary-operator trimmed '("*")) => + (lambda (match) + (let* ([index (cdr match)] + [left-text (substring-trim trimmed 0 index)] + [right-text (substring-trim + trimmed + (+ index 1) + (string-length trimmed))] + [left (python-cp-expression-value + left-text + bindings + source + before-offset)] + [right (python-cp-expression-value + right-text + bindings + source + before-offset)]) + (cond + [(and (string? left) (number? right)) + (python-cp-repeat-string left right)] + [(and (number? left) (string? right)) + (python-cp-repeat-string right left)] + [(and (string? left) + (python-cp-missing? right) + (python-cp-simple-identifier-text? right-text)) + left] + [(and (python-cp-missing? left) + (python-cp-simple-identifier-text? left-text) + (string? right)) + right] + [(and (number? left) (number? right)) (* left right)] + [else python-cp-missing-value])))] + [(python-simple-identifier? trimmed) + (python-cp-binding-ref bindings trimmed)] + [else python-cp-missing-value]))) + (def python-simple-assignment-regex + "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") + (def (regex-fold-matches pattern source proc seed) + (let ([rx (re pattern)] [len (string-length source)]) + (let loop ([start 0] [acc seed]) + (if (> start len) + acc + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next (proc match acc))) + acc)))))) + (def (python-cp-bindings-before source before-offset) + (regex-fold-matches + python-simple-assignment-regex + (substring source 0 before-offset) + (lambda (match acc) + (let* ([name (re-match-group match 2)] + [expr (re-match-group match 3)] + [value (python-cp-expression-value + expr + acc + source + (re-match-start match))]) + (if (python-cp-missing? value) + (python-cp-remove-binding acc name) + (cons + (cons name value) + (python-cp-remove-binding acc name))))) + '())) + (define python-cp-any-string-value + (list 'python-cp-any-string-value)) + (def (python-cp-any-string? value) + (eq? value python-cp-any-string-value)) + (def (python-cp-stringish? value) + (or (string? value) (python-cp-any-string? value))) + (def (python-cp-set-binding bindings name value) + (cons + (cons name value) + (python-cp-remove-binding bindings name))) + (def (python-cp-flow-binding-ref bindings name) + (let ([found (assoc name bindings)]) + (if found (cdr found) python-cp-missing-value))) + (def (python-cp-flow-repeat-string value count) + (cond + [(python-cp-any-string? value) python-cp-any-string-value] + [(string? value) (python-cp-repeat-string value count)] + [else python-cp-missing-value])) + (def (python-cp-flow-expression-value + expr + bindings + source + before-offset) + (let ([trimmed (string-trim expr)]) + (cond + [(string=? trimmed "") python-cp-missing-value] + [(outer-pair? trimmed #\( #\)) + (python-cp-flow-expression-value + (substring trimmed 1 (- (string-length trimmed) 1)) + bindings + source + before-offset)] + [(python-string-literal? trimmed) + (or (python-constant-string-literal-value + trimmed + source + before-offset) + python-cp-missing-value)] + [(or (string=? trimmed "True") (string=? trimmed "true")) + #t] + [(or (string=? trimmed "False") (string=? trimmed "false")) + #f] + [(parse-number-literal trimmed #f) => values] + [(find-top-level-binary-operator trimmed '("+")) => + (lambda (match) + (let* ([index (cdr match)] + [left (python-cp-flow-expression-value + (substring-trim trimmed 0 index) + bindings + source + before-offset)] + [right (python-cp-flow-expression-value + (substring-trim + trimmed + (+ index 1) + (string-length trimmed)) + bindings + source + before-offset)]) + (cond + [(or (python-cp-missing? left) + (python-cp-missing? right)) + python-cp-missing-value] + [(and (string? left) (string? right)) + (string-append left right)] + [(and (python-cp-stringish? left) + (python-cp-stringish? right)) + python-cp-any-string-value] + [(and (number? left) (number? right)) (+ left right)] + [else python-cp-missing-value])))] + [(find-top-level-binary-operator trimmed '("*")) => + (lambda (match) + (let* ([index (cdr match)] + [left-text (substring-trim trimmed 0 index)] + [right-text (substring-trim + trimmed + (+ index 1) + (string-length trimmed))] + [left (python-cp-flow-expression-value + left-text + bindings + source + before-offset)] + [right (python-cp-flow-expression-value + right-text + bindings + source + before-offset)]) + (cond + [(and (python-cp-stringish? left) (number? right)) + (python-cp-flow-repeat-string left right)] + [(and (number? left) (python-cp-stringish? right)) + (python-cp-flow-repeat-string right left)] + [(and (python-cp-stringish? left) + (python-cp-missing? right) + (python-cp-simple-identifier-text? right-text)) + python-cp-any-string-value] + [(and (python-cp-missing? left) + (python-cp-simple-identifier-text? left-text) + (python-cp-stringish? right)) + python-cp-any-string-value] + [(and (number? left) (number? right)) (* left right)] + [else python-cp-missing-value])))] + [(python-simple-identifier? trimmed) + (python-cp-flow-binding-ref bindings trimmed)] + [else python-cp-missing-value]))) + (def (python-cp-env-names bindings) + (let loop ([xs bindings] [seen '()] [acc '()]) + (cond + [(null? xs) (reverse acc)] + [(member (caar xs) seen) (loop (cdr xs) seen acc)] + [else + (loop + (cdr xs) + (cons (caar xs) seen) + (cons (caar xs) acc))]))) + (def (python-cp-merge-values left right) + (cond + [(or (python-cp-missing? left) (python-cp-missing? right)) + python-cp-missing-value] + [(equal? left right) left] + [(and (python-cp-stringish? left) + (python-cp-stringish? right)) + python-cp-any-string-value] + [else python-cp-missing-value])) + (def (python-cp-merge-envs left right) + (let ([names (python-cp-env-names (append left right))]) + (let loop ([xs names] [acc '()]) + (if (null? xs) + (reverse acc) + (let* ([name (car xs)] + [value (python-cp-merge-values + (python-cp-flow-binding-ref left name) + (python-cp-flow-binding-ref right name))]) + (loop + (cdr xs) + (if (python-cp-missing? value) + acc + (cons (cons name value) acc)))))))) + (def (python-cp-merge-env-list envs fallback) + (cond + [(null? envs) fallback] + [(null? (cdr envs)) (car envs)] + [else + (let loop ([remaining (cdr envs)] [merged (car envs)]) + (if (null? remaining) + merged + (loop + (cdr remaining) + (python-cp-merge-envs merged (car remaining)))))])) + (def (python-cp-next-line-start source line-start) + (let* ([len (string-length source)] + [line-end (line-end-after source line-start)]) + (if (< line-end len) (+ line-end 1) (+ len 1)))) + (def (python-cp-trim-comment text) + (let ([hash (char-index-from + text + #\# + 0 + (string-length text))]) + (string-trim (if hash (substring text 0 hash) text)))) + (def (python-cp-assignment-line-value + source + line-start + line-end + bindings) + (let ([info (python-line-assignment-info + source + line-start + line-end)]) + (and info + (let* ([rhs (python-cp-trim-comment + (alist-ref/default info 'rhs ""))] + [value (python-cp-flow-expression-value + rhs + bindings + source + (alist-ref/default info 'first line-start))]) + (list + (cons 'name (alist-ref/default info 'name "")) + (cons 'value value)))))) + (def (python-cp-augmented-assignment-line-name + source + line-start + line-end) + (let* ([first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*)[ \\t]*[+\\-*/%]=") + line + 0)]) + (and match (re-match-group match 1)))) + (def (python-cp-assigned-names-in-range source start end) + (let loop ([line-start start] [acc '()]) + (if (>= line-start end) + (reverse acc) + (let* ([line-end (min (line-end-after source line-start) + end)] + [info (python-line-assignment-info + source + line-start + line-end)] + [name (and info (alist-ref/default info 'name #f))] + [next (python-cp-next-line-start source line-start)]) + (loop + next + (if (and name (not (member name acc))) + (cons name acc) + acc)))))) + (def (python-cp-remove-names bindings names) + (let loop ([xs names] [env bindings]) + (if (null? xs) + env + (loop (cdr xs) (python-cp-remove-binding env (car xs)))))) + (def (python-cp-same-indent-clause-start source start end + indent pred) + (let loop ([line-start start]) + (and (< line-start end) + (let* ([line-end (min (line-end-after source line-start) + end)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [line-indent (- first line-start)] + [next (python-cp-next-line-start source line-start)]) + (if (and (= line-indent indent) (pred line)) + line-start + (loop next)))))) + (def (python-cp-else-line? line) + (string=? (string-trim line) "else:")) + (def (python-cp-try-clause-headers + source + try-start + try-end + indent) + (let ([body-start (python-cp-next-line-start + source + try-start)]) + (let loop ([line-start body-start] + [acc (list + (list + (cons 'kind 'try) + (cons 'start try-start)))]) + (if (>= line-start try-end) + (reverse acc) + (let* ([line-end (min (line-end-after source line-start) + try-end)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [line-indent (- first line-start)] + [kind (and (= line-indent indent) + (python-cp-try-clause-kind line))] + [next (python-cp-next-line-start source line-start)]) + (loop + next + (if (and kind (not (eq? kind 'try))) + (cons + (list (cons 'kind kind) (cons 'start line-start)) + acc) + acc))))))) + (def (python-cp-try-clause-entries + source + try-start + try-end + indent) + (let ([headers (python-cp-try-clause-headers + source + try-start + try-end + indent)]) + (let loop ([remaining headers] [acc '()]) + (if (null? remaining) + (reverse acc) + (let* ([header (car remaining)] + [next-header (and (not (null? (cdr remaining))) + (cadr remaining))] + [header-start (alist-ref/default + header + 'start + try-start)] + [body-start (python-cp-next-line-start + source + header-start)] + [body-end (if next-header + (alist-ref/default + next-header + 'start + try-end) + try-end)]) + (loop + (cdr remaining) + (cons + (list + (cons 'kind (alist-ref/default header 'kind #f)) + (cons 'start header-start) + (cons 'body-start body-start) + (cons 'body-end body-end)) + acc))))))) + (def (python-cp-entries-with-kind entries kind) + (sg-filter + (lambda (entry) + (eq? (alist-ref/default entry 'kind #f) kind)) + entries)) + (def (python-cp-entry-with-kind entries kind) + (let ([matches (python-cp-entries-with-kind entries kind)]) + (and (not (null? matches)) (car matches)))) + (def (python-cp-try-throw-kind source start end) + (let ([body (substring source start end)]) + (cond + [(string-find-substring body "raise ") 'definite] + [(string-find-substring + body + "any_function_call_may_raise()") + 'may] + [else 'none]))) + (def (python-cp-flow-analyze-entry source entry env) + (python-cp-flow-analyze-range + source + (alist-ref/default entry 'body-start 0) + (alist-ref/default entry 'body-end 0) + env)) + (def (python-cp-flow-analyze-if source first line-start + block-end range-end env) + (let* ([indent (- first line-start)] + [body-start (python-cp-next-line-start source line-start)] + [else-start (python-cp-same-indent-clause-start source body-start block-end indent + python-cp-else-line?)] + [then-end (or else-start block-end)] + [then-env (python-cp-flow-analyze-range + source + body-start + then-end + env)] + [else-env (if else-start + (python-cp-flow-analyze-range + source + (python-cp-next-line-start source else-start) + block-end + env) + env)]) + (cond + [(and (< range-end block-end) + (or (not else-start) (<= range-end else-start))) + then-env] + [(and else-start (< range-end block-end)) else-env] + [else (python-cp-merge-envs then-env else-env)]))) + (def (python-cp-flow-analyze-try source first line-start + block-end env) + (let* ([indent (- first line-start)] + [entries (python-cp-try-clause-entries + source + line-start + block-end + indent)] + [try-entry (python-cp-entry-with-kind entries 'try)] + [except-entries (python-cp-entries-with-kind + entries + 'except)] + [else-entry (python-cp-entry-with-kind entries 'else)] + [finally-entry (python-cp-entry-with-kind entries 'finally)] + [try-env (if try-entry + (python-cp-flow-analyze-entry + source + try-entry + env) + env)] + [throw-kind (if try-entry + (python-cp-try-throw-kind + source + (alist-ref/default try-entry 'body-start 0) + (alist-ref/default try-entry 'body-end 0)) + 'none)] + [normal-env (and (not (eq? throw-kind 'definite)) + (if else-entry + (python-cp-flow-analyze-entry + source + else-entry + try-env) + try-env))] + [except-envs (map (lambda (entry) + (python-cp-flow-analyze-entry + source + entry + env)) + except-entries)] + [merged (case throw-kind + [(definite) + (python-cp-merge-env-list except-envs env)] + [(none) normal-env] + [else + (python-cp-merge-env-list + (if normal-env + (cons normal-env except-envs) + except-envs) + env)])] + [with-finally (if finally-entry + (python-cp-flow-analyze-entry + source + finally-entry + merged) + merged)]) + with-finally)) + (def (python-cp-flow-analyze-assignment + source + line-start + line-end + env) + (let ([augmented (python-cp-augmented-assignment-line-name + source + line-start + line-end)]) + (if augmented + (python-cp-remove-binding env augmented) + (let ([assignment (python-cp-assignment-line-value + source + line-start + line-end + env)]) + (if assignment + (let ([name (alist-ref/default assignment 'name "")] + [value (alist-ref/default + assignment + 'value + python-cp-missing-value)]) + (if (python-cp-missing? value) + (python-cp-remove-binding env name) + (python-cp-set-binding env name value))) + env))))) + (def (python-cp-flow-analyze-range source start end env) + (let loop ([line-start start] [bindings env]) + (if (>= line-start end) + bindings + (let* ([line-end (min (line-end-after source line-start) + end)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [trimmed (string-trim line)] + [next (python-cp-next-line-start source line-start)]) + (cond + [(or (string=? trimmed "") + (sg-string-prefix? "#" trimmed)) + (loop next bindings)] + [(or (sg-string-prefix? "def " trimmed) + (sg-string-prefix? "class " trimmed)) + (let ([block-end (or (python-block-end source first #f) + next)]) + (if (> block-end end) + (python-cp-flow-analyze-range + source + next + end + bindings) + (loop block-end bindings)))] + [(sg-string-prefix? "if " trimmed) + (let ([block-end (or (python-block-end source first #f) + next)]) + (loop + block-end + (python-cp-flow-analyze-if source first line-start + block-end end bindings)))] + [(sg-string-prefix? "while " trimmed) + (let* ([block-end (or (python-block-end source first #f) + next)] + [assigned (python-cp-assigned-names-in-range + source + next + (min block-end end))]) + (loop + block-end + (python-cp-remove-names bindings assigned)))] + [(string=? trimmed "try:") + (let ([block-end (or (python-try-statement-end + source + first) + (python-block-end source first #f) + next)]) + (loop + block-end + (python-cp-flow-analyze-try source first line-start + (min block-end end) bindings)))] + [else + (loop + next + (python-cp-flow-analyze-assignment + source + line-start + line-end + bindings))]))))) + (def (python-cp-flow-env-before source offset) + (python-cp-flow-analyze-range source 0 offset '()))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -17,13 +17,17 @@ (tree-sitter tree-sitter) (semgrep lang) (semgrep rule) (semgrep result) (semgrep result builders) (semgrep result extras) (semgrep result findings) - (semgrep engine comparison) (semgrep engine generic-scan) + (only (semgrep engine comparison) comparison-missing? + comparison-value->string constant-bindings-before + comparison-value metavariable-comparison-satisfied?) + (semgrep engine generic-scan) (semgrep engine js-constructor-scan) (semgrep engine js-eval-scan) (semgrep engine js-vardef-scan) (semgrep engine js-decorator-scan) (semgrep engine markup-scan) (semgrep engine py-constant-scan) + (semgrep engine py-constant-prop) (semgrep engine py-string-eval) (semgrep engine py-string-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) @@ -12601,662 +12605,6 @@ (+ line-end 1) (+ len 1))]) (loop next (if finding (cons finding acc) acc)))))))) - (define python-cp-missing-value - (list 'python-cp-missing-value)) - (def (python-cp-missing? value) - (eq? value python-cp-missing-value)) - (def (python-cp-binding-ref bindings name) - (let ([found (assoc name bindings)]) - (if found (cdr found) python-cp-missing-value))) - (def (python-cp-remove-binding bindings name) - (let loop ([xs bindings]) - (cond - [(null? xs) '()] - [(string=? (caar xs) name) (loop (cdr xs))] - [else (cons (car xs) (loop (cdr xs)))]))) - (def (python-cp-repeat-string text count) - (if (and (number? count) (<= 0 count 256)) - (let loop ([i count] [acc '()]) - (if (= i 0) - (join-strings acc)