Extract Semgrep Python constant-propagation scanners
ober
3f76c639cc6a292a0803d2deec7434e3f50f27bd
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -498,6 +498,8 @@ Completed in the repo: `src/semgrep/engine/py-string-eval.ss` - extracted Python constant-propagation evaluation and flow-analysis core into `src/semgrep/engine/py-constant-prop.ss` + - extracted Python constant-propagation pattern-specific fallback scanners + into `src/semgrep/engine/py-cp-scan.ss` - extracted shared comparison evaluation, constant-binding resolution, and metavariable-comparison predicate handling into `src/semgrep/engine/comparison.ss` @@ -513,7 +515,10 @@ Validation at this checkpoint: Remaining Phase 0 work: - Continue splitting `src/semgrep/scan.ss` into engine, targeting, text-mode, - structural dispatch, taint, and result/output-adjacent modules. + structural dispatch, taint, and result/output-adjacent modules. The Python + constant-propagation fallback surface is now out; the next meaningful cuts + are the remaining Python f-string/special-case scanners and the larger + language dispatch and taint sections. - Run broader upstream sweeps and grow the expected-fail baselines from real sampled data instead of one narrow seed run. new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-cp-scan.sls @@ -0,0 +1,1069 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-cp-scan.ss + +(library (semgrep engine py-cp-scan) + (export scan-python-cp-wildcard-string-call-pattern + scan-python-cp-self-equality-pattern + scan-python-cp-bare-string-pattern + scan-python-cp-string-call-pattern + scan-python-cp-return-string-mvar-pattern + scan-python-cp-yield-number-pattern + scan-python-cp-format-pattern + scan-python-cp-subscript-string-pattern + scan-python-cp-set-cookie-pattern + scan-python-cp-password-concat-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (only + (semgrep engine comparison) + find-top-level-binary-operator + substring-trim + skip-whitespace) + (only (semgrep engine generic-scan) regex-escape-string) + (semgrep engine py-constant-prop) + (semgrep engine regex-support) (semgrep engine text-support) + (only (semgrep result findings) normalize-metavariable-name) + (semgrep util literals)) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (sg-string-prefix? prefix s) + (let ([prefix-len (string-length prefix)] + [len (string-length s)]) + (and (<= prefix-len len) + (string=? (substring s 0 prefix-len) prefix)))) + (def (sg-string-suffix? suffix s) + (let ([suffix-len (string-length suffix)] + [len (string-length s)]) + (and (<= suffix-len len) + (string=? (substring s (- len suffix-len) len) suffix)))) + (def (identifier-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$))) + (def (identifier-start-char? ch) + (or (char-alphabetic? ch) (char=? ch #\_) (char=? ch #\$))) + (def (scan-forward-identifier-end source start) + (let ([len (string-length source)]) + (let loop ([i start]) + (if (and (< i len) (identifier-char? (string-ref source i))) + (loop (+ i 1)) + i)))) + (def (code-string-quote? ch) + (or (char=? ch #\") (char=? ch #\') (char=? ch #\`))) + (def (parse-code-string-literal-loop source len quote + content-start i escaped?) + (cond + [(>= i len) #f] + [escaped? + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #f)] + [(char=? (string-ref source i) #\\) + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #t)] + [(and (char=? quote #\`) + (< (+ i 1) len) + (char=? (string-ref source i) #\$) + (char=? (string-ref source (+ i 1)) #\{)) + #f] + [(char=? (string-ref source i) quote) + (cons (substring source content-start i) (+ i 1))] + [else + (parse-code-string-literal-loop source len quote + content-start (+ i 1) #f)])) + (def (parse-code-string-literal-at source start) + (let ([len (string-length source)]) + (if (< start len) + (let ([quote (string-ref source start)]) + (and (code-string-quote? quote) + (parse-code-string-literal-loop source len quote + (+ start 1) (+ start 1) #f))) + #f))) + (def (skip-horizontal-backward source i) + (let loop ([j i]) + (if (and (>= j 0) + (let ([ch (string-ref source j)]) + (or (char=? ch #\space) (char=? ch #\tab)))) + (loop (- j 1)) + j))) + (def (line-start-before source offset) + (let loop ([i (- offset 1)]) + (cond + [(< i 0) 0] + [(char=? (string-ref source i) #\newline) (+ i 1)] + [else (loop (- i 1))]))) + (def (previous-nonspace-index source index) + (let loop ([i (- index 1)]) + (cond + [(< i 0) #f] + [(char-whitespace? (string-ref source i)) (loop (- i 1))] + [else i]))) + (def (last-char-index-before source ch start end) + (let loop ([i (- end 1)]) + (cond + [(< i start) #f] + [(char=? (string-ref source i) ch) i] + [else (loop (- i 1))]))) + (def (skip-pattern-horizontal source i end) + (let loop ([j i]) + (if (and (< j end) + (let ([ch (string-ref source j)]) + (or (char=? ch #\space) + (char=? ch #\tab) + (char=? ch #\return) + (char=? ch #\newline)))) + (loop (+ j 1)) + j))) + (def (split-top-level-commas source) + (let ([len (string-length source)]) + (let loop ([i 0] + [start 0] + [depth 0] + [state 'normal] + [escaped? #f] + [acc '()]) + (cond + [(>= i len) + (reverse (cons (substring-trim source start len) acc))] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") + (loop (+ i 1) start depth 'double #f acc)] + [(char=? ch #\') + (loop (+ i 1) start depth 'single #f acc)] + [(char=? ch #\`) + (loop (+ i 1) start depth 'backtick #f acc)] + [(or (char=? ch #\() (char=? ch #\[)) + (loop (+ i 1) start (+ depth 1) state #f acc)] + [(or (char=? ch #\)) (char=? ch #\])) + (loop (+ i 1) start (max 0 (- depth 1)) state #f acc)] + [(and (= depth 0) (char=? ch #\,)) + (loop (+ i 1) (+ i 1) depth state #f + (cons (substring-trim source start i) acc))] + [else (loop (+ i 1) start depth state #f acc)]))] + [escaped? (loop (+ i 1) start depth state #f acc)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) start depth state #t acc)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) start depth 'normal #f acc)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) start depth 'normal #f acc)] + [(and (eq? state 'backtick) + (char=? (string-ref source i) #\`)) + (loop (+ i 1) start depth 'normal #f acc)] + [else (loop (+ i 1) start depth state #f acc)])))) + (def (find-matching-close-paren source open-index) + (let ([len (string-length source)]) + (let loop ([i open-index] + [depth 0] + [state 'normal] + [escaped? #f]) + (cond + [(>= i len) #f] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") (loop (+ i 1) depth 'double #f)] + [(char=? ch #\') (loop (+ i 1) depth 'single #f)] + [(char=? ch #\`) (loop (+ i 1) depth 'backtick #f)] + [(char=? ch #\() (loop (+ i 1) (+ depth 1) state #f)] + [(char=? ch #\)) + (if (= depth 1) + (+ i 1) + (loop (+ i 1) (max 0 (- depth 1)) state #f))] + [else (loop (+ i 1) depth state #f)]))] + [escaped? (loop (+ i 1) depth state #f)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) depth state #t)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'backtick) + (char=? (string-ref source i) #\`)) + (loop (+ i 1) depth 'normal #f)] + [else (loop (+ i 1) depth state #f)])))) + (def (find-matching-close-bracket source open-index) + (let ([len (string-length source)]) + (let loop ([i open-index] + [depth 0] + [state 'normal] + [escaped? #f]) + (cond + [(>= i len) #f] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") (loop (+ i 1) depth 'double #f)] + [(char=? ch #\') (loop (+ i 1) depth 'single #f)] + [(char=? ch #\[) (loop (+ i 1) (+ depth 1) state #f)] + [(char=? ch #\]) + (if (= depth 1) + (+ i 1) + (loop (+ i 1) (max 0 (- depth 1)) state #f))] + [else (loop (+ i 1) depth state #f)]))] + [escaped? (loop (+ i 1) depth state #f)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) depth state #t)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) depth 'normal #f)] + [else (loop (+ i 1) depth state #f)])))) + (def (identifier-boundary-before? source index) + (or (= index 0) + (not (identifier-char? (string-ref source (- index 1)))))) + (def (python-simple-identifier? text) + (let ([len (string-length text)]) + (and (> len 0) + (let ([first (string-ref text 0)]) + (or (char-alphabetic? first) (char=? first #\_))) + (let loop ([i 1]) + (cond + [(= i len) #t] + [(identifier-char? (string-ref text i)) (loop (+ i 1))] + [else #f]))))) + (def (python-previous-horizontal-nonspace-index + source + index) + (let loop ([i (- index 1)]) + (cond + [(< i 0) #f] + [(char=? (string-ref source i) #\newline) #f] + [(let ([ch (string-ref source i)]) + (or (char=? ch #\space) (char=? ch #\tab))) + (loop (- i 1))] + [else i]))) + (def (ascii-string? text) + (let ([len (string-length text)]) + (let loop ([i 0]) + (or (= i len) + (and (<= (char->integer (string-ref text i)) 127) + (loop (+ i 1))))))) + (def (python-line-has-comment-before? source index) + (let ([line-start (line-start-before source index)]) + (let loop ([i line-start]) + (cond + [(>= i index) #f] + [(char=? (string-ref source i) #\#) #t] + [else (loop (+ i 1))])))) + (def (python-previous-code-line source line-start) + (let loop ([end (- line-start 1)]) + (and (>= end 0) + (let* ([prev-start (line-start-before source end)] + [prev-end (line-end-after source prev-start)] + [first (line-first-nonspace + source + prev-start + prev-end)] + [text (string-trim + (substring source first prev-end))]) + (if (or (string=? text "") (sg-string-prefix? "#" text)) + (loop (- prev-start 1)) + text))))) + (def (python-cp-label-content-only-literal? source start) + (let* ([line-start (line-start-before source start)] + [previous (python-previous-code-line source line-start)]) + (and previous (sg-string-suffix? "\\" previous)))) + (def (python-cp-expression-token-context? source start end) + (let* ([prev (previous-nonspace-index source start)] + [line-start (line-start-before source start)] + [prefix (substring source line-start start)] + [next (skip-horizontal-forward source end)] + [next-char (and (< next (string-length source)) + (string-ref source next))]) + (and (not (and next-char (char=? next-char #\=))) + (or (and prev + (let ([ch (string-ref source prev)]) + (or (char=? ch #\=) + (char=? ch #\[) + (char=? ch #\() + (char=? ch #\,) + (char=? ch #\:)))) + (sg-string-suffix? "return " prefix))))) + (def (python-cp-line-indent-at source offset) + (let* ([line-start (line-start-before source offset)] + [line-end (line-end-after source line-start)] + [first (line-first-nonspace source line-start line-end)]) + (- first line-start))) + (def (regex-fold-matches pattern source proc seed) + (let ([rx (re pattern)] [len (string-length source)]) + (let loop ([start 0] [acc seed]) + (if (> start len) + acc + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next (proc match acc))) + acc)))))) + (def (python-cp-last-assignment-deeper-than? + source + name + before-offset) + (let ([pattern (string-append + "(^|\\n)[ \\t]*" + (regex-escape-string name) + "[ \\t]*=")] + [call-indent (python-cp-line-indent-at + source + before-offset)]) + (regex-fold-matches + pattern + (substring source 0 before-offset) + (lambda (match acc) + (let* ([start (re-match-start match)] + [line-start (if (and (< start (string-length source)) + (char=? + (string-ref source start) + #\newline)) + (+ start 1) + start)] + [line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [indent (- first line-start)]) + (> indent call-indent))) + #f))) + (def (python-cp-augmented-assignment-before? + source + name + before-offset) + (let ([pattern (string-append + "(^|\\n)[ \\t]*" + (regex-escape-string name) + "[ \\t]*[+\\-*/%]=")]) + (if (re-search + (re pattern) + (substring source 0 before-offset) + 0) + #t + #f))) + (def (python-cp-unsafe-identifier-use? + source + expr + before-offset) + (let ([name (string-trim expr)]) + (and (python-simple-identifier? name) + (or (python-cp-augmented-assignment-before? + source + name + before-offset) + (python-cp-last-assignment-deeper-than? + source + name + before-offset))))) + (def (python-cp-dotted-call-string-pattern-spec pattern) + (let* ([trimmed (string-trim pattern)] + [match (re-search + (re "^([A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*)[ \\t\\r\\n]*\\(") + trimmed + 0)]) + (and match + (let* ([function-name (re-match-group match 1)] + [open (- (re-match-end match) 1)] + [len (string-length trimmed)] + [arg-start0 (skip-pattern-horizontal + trimmed + (+ open 1) + len)] + [f-prefix? (and (< arg-start0 len) + (let ([ch (string-ref + trimmed + arg-start0)]) + (or (char=? ch #\f) + (char=? ch #\F))))] + [arg-start (if f-prefix? + (+ arg-start0 1) + arg-start0)]) + (and (< arg-start len) + (let ([quote (string-ref trimmed arg-start)]) + (and (or (char=? quote #\") (char=? quote #\')) + (let* ([close-paren (last-char-index-before + trimmed + #\) + (+ arg-start 1) + len)] + [quote-end (and close-paren + (last-char-index-before + trimmed + quote + (+ arg-start 1) + close-paren))] + [tail-start (and quote-end + close-paren + (skip-pattern-horizontal + trimmed + (+ quote-end 1) + close-paren))]) + (and close-paren + quote-end + (= tail-start close-paren) + (let ([content (substring + trimmed + (+ arg-start 1) + quote-end)]) + (list + (cons "function" function-name) + (cons "expected" content) + (cons + "wildcard-string" + (string=? + content + "..."))))))))))))) + (def (python-cp-wildcard-string-call-pattern? pattern) + (let ([spec (python-cp-dotted-call-string-pattern-spec + pattern)]) + (and spec + (alist-ref/default spec "wildcard-string" #f) + spec))) + (def (scan-python-cp-wildcard-string-call-pattern rule path source pattern initial-bindings) + (let ([spec (python-cp-wildcard-string-call-pattern? + pattern)]) + (and spec + (let* ([function-name (alist-ref/default + spec + "function" + "")] + [needle (string-append function-name "(")] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([index (string-find-substring-from + source + needle + start)]) + (if (not index) + (nonempty-findings (reverse acc)) + (let* ([open (+ index + (string-length function-name))] + [close (find-matching-close-paren + source + open)] + [args (and close + (split-top-level-commas + (substring + source + (+ open 1) + (- close 1))))] + [arg (and args + (null? (cdr args)) + (car args))] + [value (and arg + (python-cp-flow-expression-value + arg + (python-cp-flow-env-before + source + index) + source + index))] + [finding (and close + (python-cp-stringish? + value) + (identifier-boundary-before? + source + index) + (finding-for-range-with-bindings rule path source index + close + initial-bindings))] + [next (if close + (max (+ index 1) close) + (+ index 1))]) + (loop + next + (if finding + (cons finding acc) + acc))))))))))) + (def (python-cp-self-equality-pattern? pattern) + (string=? (string-trim pattern) "$X == $X")) + (def (scan-python-cp-self-equality-pattern rule path source pattern initial-bindings) + (and (python-cp-self-equality-pattern? pattern) + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [trimmed (string-trim line)] + [match (and (not (sg-string-prefix? + "#" + trimmed)) + (find-top-level-binary-operator + line + '("==")))] + [op-index (and match (cdr match))] + [left (and op-index + (substring-trim line 0 op-index))] + [right (and op-index + (substring-trim + line + (+ op-index 2) + (string-length line)))] + [env (and left + (python-cp-flow-env-before + source + first))] + [left-value (and env + (python-cp-flow-expression-value + left + env + source + first))] + [right-value (and env + (python-cp-flow-expression-value + right + env + source + first))] + [finding (and left-value + right-value + (not (python-cp-missing? + left-value)) + (not (python-cp-missing? + right-value)) + (not (python-cp-any-string? + left-value)) + (not (python-cp-any-string? + right-value)) + (equal? left-value right-value) + (finding-for-range-with-bindings rule path source first line-end + initial-bindings))] + [next (python-cp-next-line-start + source + line-start)]) + (loop next (if finding (cons finding acc) acc)))))))) + (def (python-cp-bare-string-pattern-value pattern) + (let ([trimmed (string-trim pattern)]) + (and (quoted-string? trimmed) + (let ([value (unquote-string trimmed)]) + (and (ascii-string? value) value))))) + (def (scan-python-cp-bare-string-pattern rule path source + pattern initial-bindings) + (let ([expected (python-cp-bare-string-pattern-value + pattern)]) + (and expected + (let ([len (string-length source)]) + (let loop ([i 0] [acc '()]) + (cond + [(>= i len) (nonempty-findings (reverse acc))] + [(char=? (string-ref source i) #\#) + (let ([line-end (line-end-after source i)]) + (loop + (if (< line-end len) (+ line-end 1) (+ len 1)) + acc))] + [(parse-code-string-literal-at source i) => + (lambda (literal) + (let* ([value (car literal)] + [literal-end (cdr literal)] + [content-only? (python-cp-label-content-only-literal? + source + i)] + [start (if content-only? (+ i 1) i)] + [end (if content-only? + (- literal-end 1) + literal-end)] + [finding (and (not (python-line-has-comment-before? + source + i)) + (string=? value expected) + (finding-for-range-with-bindings rule path source start end + initial-bindings))]) + (loop + literal-end + (if finding (cons finding acc) acc))))] + [(identifier-start-char? (string-ref source i)) + (let* ([end (scan-forward-identifier-end source i)] + [name (substring source i end)] + [env (python-cp-flow-env-before source i)] + [value (python-cp-flow-binding-ref env name)] + [finding (and (not (python-line-has-comment-before? + source + i)) + (python-cp-expression-token-context? + source + i + end) + (string? value) + (string=? value expected) + (finding-for-range-with-bindings rule path source i end + initial-bindings))]) + (loop end (if finding (cons finding acc) acc)))] + [else (loop (+ i 1) acc)])))))) + (def (scan-python-cp-string-call-pattern rule path source + pattern initial-bindings) + (let ([spec (python-cp-dotted-call-string-pattern-spec + pattern)]) + (and spec + (let* ([function-name (alist-ref/default + spec + "function" + "")] + [expected (alist-ref/default spec "expected" "")] + [wildcard? (alist-ref/default + spec + "wildcard-string" + #f)] + [needle (string-append function-name "(")] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([index (string-find-substring-from + source + needle + start)]) + (if (not index) + (nonempty-findings (reverse acc)) + (let* ([open (+ index + (string-length function-name))] + [close (find-matching-close-paren + source + open)] + [args (and close + (split-top-level-commas + (substring + source + (+ open 1) + (- close 1))))] + [arg (and args + (null? (cdr args)) + (car args))] + [value (and arg + (not (python-cp-unsafe-identifier-use? + source + arg + index)) + (python-cp-expression-value + arg + (python-cp-bindings-before + source + index) + source + index))] + [matches? (and (string? value) + (or wildcard? + (string=? + value + expected)))] + [finding (and close + matches? + (identifier-boundary-before? + source + index) + (finding-for-range-with-bindings rule path source index + close + initial-bindings))] + [next (if close + (max (+ index 1) close) + (+ index 1))]) + (loop + next + (if finding + (cons finding acc) + acc))))))))))) + (def (python-cp-return-string-mvar-pattern-spec pattern) + (let* ([trimmed (string-trim pattern)] [prefix "return "]) + (and (sg-string-prefix? prefix trimmed) + (let ([arg (string-trim + (substring + trimmed + (string-length prefix) + (string-length trimmed)))]) + (and (>= (string-length arg) 4) + (char=? (string-ref arg 0) #\") + (char=? + (string-ref arg (- (string-length arg) 1)) + #\") + (let ([body (substring + arg + 1 + (- (string-length arg) 1))]) + (and (sg-string-prefix? "$" body) + (normalize-metavariable-name body)))))))) + (def (scan-python-cp-return-string-mvar-pattern rule path source pattern initial-bindings) + (let ([mvar (python-cp-return-string-mvar-pattern-spec + pattern)]) + (and mvar + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [expr (and (sg-string-prefix? "return " line) + (string-trim + (substring + line + (string-length "return ") + (string-length line))))] + [value (and expr + (python-cp-expression-value + expr + (python-cp-bindings-before + source + first) + source + first))] + [expr-start (and expr + (string-find-substring-from + line + expr + (string-length + "return ")))] + [binding (and (string? value) + expr-start + (make-regex-capture-binding mvar value source + (+ first expr-start) + (+ first + expr-start + (string-length expr))))] + [finding (and binding + (finding-for-range-with-bindings rule path source first + line-end + (append + initial-bindings + (list + (cons mvar binding)))))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop + next + (if finding (cons finding acc) acc))))))))) + (def (python-cp-yield-number-pattern-value pattern) + (let ([trimmed (string-trim pattern)] [prefix "yield "]) + (and (sg-string-prefix? prefix trimmed) + (parse-number-literal + (substring + trimmed + (string-length prefix) + (string-length trimmed)) + #f)))) + (def (scan-python-cp-yield-number-pattern rule path source + pattern initial-bindings) + (let ([expected (python-cp-yield-number-pattern-value + pattern)]) + (and expected + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [expr (and (sg-string-prefix? "yield " line) + (string-trim + (substring + line + (string-length "yield ") + (string-length line))))] + [value (and expr + (python-cp-expression-value + expr + (python-cp-bindings-before + source + first) + source + first))] + [finding (and (number? value) + (= value expected) + (finding-for-range-with-bindings rule path source first + line-end initial-bindings))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop + next + (if finding (cons finding acc) acc))))))))) + (def (python-cp-format-pattern? pattern) + (string=? (string-trim pattern) "\"...\".format(...)")) + (def (python-token-start-before-dot source dot) + (let ([end (skip-horizontal-backward source (- dot 1))]) + (and (>= end 0) + (let loop ([start end]) + (if (and (> start 0) + (identifier-char? + (string-ref source (- start 1)))) + (loop (- start 1)) + (cons start (+ end 1))))))) + (def (scan-python-cp-format-pattern rule path source pattern + initial-bindings) + (and (python-cp-format-pattern? pattern) + (let ([len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([dot (string-find-substring-from + source + ".format(" + start)]) + (if (not dot) + (nonempty-findings (reverse acc)) + (let* ([receiver-range (python-token-start-before-dot + source + dot)] + [receiver (and receiver-range + (substring + source + (car receiver-range) + (cdr receiver-range)))] + [open (+ dot (string-length ".format"))] + [close (find-matching-close-paren + source + open)] + [value (and receiver + (python-cp-expression-value + receiver + (python-cp-bindings-before + source + (car receiver-range)) + source + (car receiver-range)))] + [finding (and close + (string? value) + (finding-for-range-with-bindings rule path source + (car receiver-range) close + initial-bindings))] + [next (if close + (max (+ dot 1) close) + (+ dot 1))]) + (loop + next + (if finding (cons finding acc) acc)))))))))) + (def (python-cp-subscript-string-pattern-spec pattern) + (let* ([trimmed (string-trim pattern)] + [open (string-find-substring-from trimmed "[" 0)] + [close (and open + (find-matching-close-bracket trimmed open))]) + (and open + close + (sg-string-prefix? + "$" + (string-trim (substring trimmed 0 open))) + (let ([key (string-trim + (substring trimmed (+ open 1) (- close 1)))]) + (and (quoted-string? key) (unquote-string key)))))) + (def (python-token-start-before-bracket source bracket) + (let ([end (skip-horizontal-backward source (- bracket 1))]) + (and (>= end 0) + (let loop ([start end]) + (if (and (> start 0) + (identifier-char? + (string-ref source (- start 1)))) + (loop (- start 1)) + (cons start (+ end 1))))))) + (def (scan-python-cp-subscript-string-pattern rule path source pattern initial-bindings) + (let ([expected (python-cp-subscript-string-pattern-spec + pattern)]) + (and expected + (let ([len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([open (char-index-from source #\[ start len)]) + (if (not open) + (nonempty-findings (reverse acc)) + (let* ([close (find-matching-close-bracket + source + open)] + [base-range (python-token-start-before-bracket + source + open)] + [key-expr (and close + (substring + source + (+ open 1) + (- close 1)))] + [key-value (and key-expr + (python-cp-expression-value + key-expr + (python-cp-bindings-before + source + open) + source + open))] + [finding (and close + base-range + (string? key-value) + (string=? + key-value + expected) + (finding-for-range-with-bindings rule path source + (car base-range) close + initial-bindings))] + [next (if close + (max (+ open 1) close) + (+ open 1))]) + (loop + next + (if finding + (cons finding acc) + acc))))))))))) + (def (python-cp-set-cookie-pattern? pattern) + (let ([trimmed (string-trim pattern)]) + (and (sg-string-prefix? + "flask.response.set_cookie(" + trimmed) + (string-find-substring-from trimmed "httponly=True" 0) + (string-find-substring-from trimmed "secure=True" 0)))) + (def (python-cp-keyword-argument-value args name) + (let ([needle (string-append name "=")]) + (let loop ([remaining (split-top-level-commas args)]) + (and (not (null? remaining)) + (let ([arg (string-trim (car remaining))]) + (if (sg-string-prefix? needle arg) + (string-trim + (substring + arg + (string-length needle) + (string-length arg))) + (loop (cdr remaining)))))))) + (def (scan-python-cp-set-cookie-pattern rule path source + pattern initial-bindings) + (and (python-cp-set-cookie-pattern? pattern) + (let ([len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([index (string-find-substring-from + source + ".set_cookie(" + start)]) + (if (not index) + (nonempty-findings (reverse acc)) + (let* ([name-range (python-token-start-before-dot + source + index)] + [call-start (and name-range + (car name-range))] + [open (+ index + (string-length ".set_cookie"))] + [close (find-matching-close-paren + source + open)] + [args (and close + (substring + source + (+ open 1) + (- close 1)))]