Extract Semgrep Python f-string scanners
ober
1675ebfd8e9acdbc8f35c24845fc8e7196d86def
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -500,6 +500,12 @@ Completed in the repo: into `src/semgrep/engine/py-constant-prop.ss` - extracted Python constant-propagation pattern-specific fallback scanners into `src/semgrep/engine/py-cp-scan.ss` + - extracted Python f-string, interpolated call/assignment, PEP 614 + decorator, and f-string-equivalence fallback scanners into + `src/semgrep/engine/py-fstring-scan.ss` + - exported shared Python line-assignment analysis from + `src/semgrep/engine/py-constant-prop.ss` so remaining Python fallbacks in + `scan.ss` can reuse one assignment-info implementation - extracted shared comparison evaluation, constant-binding resolution, and metavariable-comparison predicate handling into `src/semgrep/engine/comparison.ss` @@ -516,9 +522,9 @@ Remaining Phase 0 work: - Continue splitting `src/semgrep/scan.ss` into engine, targeting, text-mode, structural dispatch, taint, and result/output-adjacent modules. The Python - constant-propagation fallback surface is now out; the next meaningful cuts - are the remaining Python f-string/special-case scanners and the larger - language dispatch and taint sections. + constant-propagation and f-string/interpolated fallback surfaces are now + out; the next meaningful cuts are the remaining Python special-case scanners + plus the larger language dispatch and taint sections. - Run broader upstream sweeps and grow the expected-fail baselines from real sampled data instead of one narrow seed run. --- a/lib/semgrep/engine/py-constant-prop.sls +++ b/lib/semgrep/engine/py-constant-prop.sls @@ -8,7 +8,8 @@ python-cp-expression-value python-cp-bindings-before python-cp-set-binding python-cp-flow-binding-ref python-cp-remove-binding python-cp-flow-expression-value - python-cp-next-line-start python-cp-flow-env-before) + python-cp-next-line-start python-cp-flow-env-before + python-line-assignment-info) (import (except (chezscheme) make-hash-table hash-table? sort sort! printf fprintf format path-extension path-absolute? new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-fstring-scan.sls @@ -0,0 +1,578 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-fstring-scan.ss + +(library (semgrep engine py-fstring-scan) + (export scan-python-fstring-ellipsis-pattern + scan-python-interpolated-assignment-pattern + scan-python-interpolated-call-pattern + scan-python-pep614-decorator-pattern + scan-python-fstring-equivalence-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (only + (semgrep engine comparison) + find-top-level-binary-operator + substring-trim) + (semgrep engine py-constant-prop) + (semgrep engine regex-support) + (semgrep engine text-support)) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (sg-string-prefix? prefix s) + (let ([prefix-len (string-length prefix)] + [len (string-length s)]) + (and (<= prefix-len len) + (string=? (substring s 0 prefix-len) prefix)))) + (def (string-find-substring s needle) + (let ([needle-len (string-length needle)] + [len (string-length s)]) + (let loop ([i 0]) + (cond + [(> (+ i needle-len) len) #f] + [(string=? (substring s i (+ i needle-len)) needle) i] + [else (loop (+ i 1))])))) + (def (identifier-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$))) + (def (identifier-boundary-before? source index) + (or (= index 0) + (not (identifier-char? (string-ref source (- index 1)))))) + (def (python-simple-identifier? text) + (let ([len (string-length text)]) + (and (> len 0) + (let ([first (string-ref text 0)]) + (or (char-alphabetic? first) (char=? first #\_))) + (let loop ([i 1]) + (cond + [(= i len) #t] + [(identifier-char? (string-ref text i)) (loop (+ i 1))] + [else #f]))))) + (def (split-top-level-commas source) + (let ([len (string-length source)]) + (let loop ([i 0] + [start 0] + [depth 0] + [state 'normal] + [escaped? #f] + [acc '()]) + (cond + [(>= i len) + (reverse (cons (substring-trim source start len) acc))] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") + (loop (+ i 1) start depth 'double #f acc)] + [(char=? ch #\') + (loop (+ i 1) start depth 'single #f acc)] + [(char=? ch #\`) + (loop (+ i 1) start depth 'backtick #f acc)] + [(or (char=? ch #\() (char=? ch #\[)) + (loop (+ i 1) start (+ depth 1) state #f acc)] + [(or (char=? ch #\)) (char=? ch #\])) + (loop (+ i 1) start (max 0 (- depth 1)) state #f acc)] + [(and (= depth 0) (char=? ch #\,)) + (loop (+ i 1) (+ i 1) depth state #f + (cons (substring-trim source start i) acc))] + [else (loop (+ i 1) start depth state #f acc)]))] + [escaped? (loop (+ i 1) start depth state #f acc)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) start depth state #t acc)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) start depth 'normal #f acc)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) start depth 'normal #f acc)] + [(and (eq? state 'backtick) + (char=? (string-ref source i) #\`)) + (loop (+ i 1) start depth 'normal #f acc)] + [else (loop (+ i 1) start depth state #f acc)])))) + (def (find-matching-close-paren source open-index) + (let ([len (string-length source)]) + (let loop ([i open-index] + [depth 0] + [state 'normal] + [escaped? #f]) + (cond + [(>= i len) #f] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") (loop (+ i 1) depth 'double #f)] + [(char=? ch #\') (loop (+ i 1) depth 'single #f)] + [(char=? ch #\`) (loop (+ i 1) depth 'backtick #f)] + [(char=? ch #\() (loop (+ i 1) (+ depth 1) state #f)] + [(char=? ch #\)) + (if (= depth 1) + (+ i 1) + (loop (+ i 1) (max 0 (- depth 1)) state #f))] + [else (loop (+ i 1) depth state #f)]))] + [escaped? (loop (+ i 1) depth state #f)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) depth state #t)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'backtick) + (char=? (string-ref source i) #\`)) + (loop (+ i 1) depth 'normal #f)] + [else (loop (+ i 1) depth state #f)])))) + (def (python-fstring-literal-range-at source start limit) + (and (< (+ start 1) limit) + (let ([prefix (string-ref source start)] + [quote (string-ref source (+ start 1))]) + (and (or (char=? prefix #\f) (char=? prefix #\F)) + (or (char=? quote #\") (char=? quote #\')) + (let loop ([i (+ start 2)] [escaped? #f]) + (cond + [(>= i limit) #f] + [escaped? (loop (+ i 1) #f)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) #t)] + [(char=? (string-ref source i) quote) (+ i 1)] + [else (loop (+ i 1) #f)])))))) + (def (unique-string-list xs) + (let loop ([remaining xs] [seen '()] [acc '()]) + (cond + [(null? remaining) (reverse acc)] + [(member (car remaining) seen) + (loop (cdr remaining) seen acc)] + [else + (loop + (cdr remaining) + (cons (car remaining) seen) + (cons (car remaining) acc))]))) + (def (python-fstring-ellipsis-pattern? pattern) + (string=? (string-trim pattern) "f\"...\"")) + (def (python-fstring-interpolation-ranges source start end) + (let ([content-start (+ start 2)] [content-end (- end 1)]) + (let loop ([i content-start] [acc '()]) + (cond + [(>= i content-end) (reverse acc)] + [(char=? (string-ref source i) #\{) + (let ([close (char-index-from + source + #\} + (+ i 1) + content-end)]) + (if close + (loop (+ close 1) (cons (cons i (+ close 1)) acc)) + (loop (+ i 1) acc)))] + [else (loop (+ i 1) acc)])))) + (def (scan-python-fstring-ellipsis-pattern rule path source pattern initial-bindings) + (and (python-fstring-ellipsis-pattern? pattern) + (let ([len (string-length source)] + [include-interpolations? (not (string-find-substring + source + "\nmatch status:"))]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let find ([i start]) + (cond + [(>= i len) (nonempty-findings (reverse acc))] + [(and (or (char=? (string-ref source i) #\f) + (char=? (string-ref source i) #\F)) + (identifier-boundary-before? source i) + (python-fstring-literal-range-at + source + i + len)) => + (lambda (end) + (let* ([whole (finding-for-range-with-bindings rule path source i end + initial-bindings)] + [interpolation-findings (if include-interpolations? + (map (lambda (range) + (finding-for-range-with-bindings rule + path + source + (car range) + (cdr range) + initial-bindings)) + (python-fstring-interpolation-ranges + source + i + end)) + '())]) + (loop + end + (append + (reverse interpolation-findings) + (if whole (cons whole acc) acc)))))] + [else (find (+ i 1))]))))))) + (def (python-interpolated-assignment-pattern? pattern) + (string=? (string-trim pattern) "$X = \"...\"")) + (def (scan-python-interpolated-assignment-pattern rule path source pattern initial-bindings) + (and (python-interpolated-assignment-pattern? pattern) + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [equals (char-index-from + line + #\= + 0 + (string-length line))] + [lhs (and equals + (string-trim + (substring line 0 equals)))] + [rhs (and equals + (string-trim + (substring + line + (+ equals 1) + (string-length line))))] + [value (and lhs + rhs + (python-simple-identifier? lhs) + (python-cp-expression-value + rhs + (python-cp-bindings-before + source + first) + source + first))] + [binding (and (string? value) + (make-regex-capture-binding "X" lhs source first + (+ first (string-length lhs))))] + [finding (and binding + (finding-for-range-with-bindings rule path source first line-end + (append + initial-bindings + (list (cons "X" binding)))))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop next (if finding (cons finding acc) acc)))))))) + (def (python-interpolated-call-pattern? pattern) + (string=? (string-trim pattern) "$FUNC(\"...\")")) + (def (scan-python-interpolated-call-pattern rule path source pattern initial-bindings) + (and (python-interpolated-call-pattern? pattern) + (let ([rx (re "\\b([A-Za-z_][A-Za-z0-9_]*)[ \\t]*\\(")] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (>= start len) + (nonempty-findings (reverse acc)) + (let ([match (re-search rx source start)]) + (if (not match) + (nonempty-findings (reverse acc)) + (let* ([call-start (re-match-start match)] + [func (re-match-group match 1)] + [open (- (re-match-end match) 1)] + [close (find-matching-close-paren + source + open)] + [args (and close + (split-top-level-commas + (substring + source + (+ open 1) + (- close 1))))] + [arg (and args + (null? (cdr args)) + (car args))] + [value (and arg + (python-cp-expression-value + arg + (python-cp-bindings-before + source + call-start) + source + call-start))] + [func-binding (and (string? value) + (make-regex-capture-binding "FUNC" func source + call-start + (+ call-start + (string-length + func))))] + [finding (and close + func-binding + (finding-for-range-with-bindings rule path source + call-start close + (append + initial-bindings + (list + (cons + "FUNC" + func-binding)))))] + [next (if close + (max (+ call-start 1) close) + (re-match-end match))]) + (loop + next + (if finding (cons finding acc) acc)))))))))) + (def (python-pep614-decorator-pattern? pattern) + (string=? (string-trim pattern) "@why := $EXP")) + (def (scan-python-pep614-decorator-pattern rule path source pattern initial-bindings) + (and (python-pep614-decorator-pattern? pattern) + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [prefix "@why :="] + [match? (sg-string-prefix? prefix line)] + [expr-start (and match? + (skip-horizontal-forward + source + (+ first + (string-length prefix))))] + [binding (and expr-start + (< expr-start line-end) + (metavariable-binding-for-range + "EXP" + source + expr-start + line-end))] + [finding (and binding + (finding-for-range-with-bindings rule path source first line-end + (append + initial-bindings + (list + (cons "EXP" binding)))))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop next (if finding (cons finding acc) acc)))))))) + (def (python-fstring-equivalence-pattern-kind pattern) + (let ([trimmed (string-trim pattern)]) + (cond + [(string=? trimmed "$M = \"...\"\n...\n$Q = f\"...{$M}\"") + 'string-suffix-assignment] + [(string=? trimmed "$M = \"...\"\n...\nf\"{$M}...\"") + 'string-prefix] + [(string=? trimmed "$M = \"...\"\n...\nf\"...{$M}...\"") + 'string-anywhere] + [(string=? trimmed "$M = ... + ...\nf\"...{$M}...\"") + 'binary-anywhere] + [else #f]))) + (def (python-fstring-content-interpolation-names + source + start + end + mode) + (let loop ([i start] [acc '()]) + (cond + [(>= i end) (reverse acc)] + [(char=? (string-ref source i) #\{) + (let ([close (char-index-from source #\} (+ i 1) end)]) + (if close + (let* ([expr (string-trim + (substring source (+ i 1) close))] + [name (and (python-simple-identifier? expr) expr)] + [position-ok? (case mode + [(prefix) (= i start)] + [(suffix) (= (+ close 1) end)] + [else #t])]) + (loop + (+ close 1) + (if (and name position-ok?) (cons name acc) acc))) + (loop (+ i 1) acc)))] + [else (loop (+ i 1) acc)]))) + (def (python-fstring-line-interpolation-names + source + start + end + mode) + (let loop ([i start] [acc '()]) + (cond + [(>= i end) (unique-string-list (reverse acc))] + [(and (or (char=? (string-ref source i) #\f) + (char=? (string-ref source i) #\F)) + (identifier-boundary-before? source i) + (python-fstring-literal-range-at source i end)) => + (lambda (fstring-end) + (loop + fstring-end + (append + (reverse + (python-fstring-content-interpolation-names + source + (+ i 2) + (- fstring-end 1) + mode)) + acc)))] + [else (loop (+ i 1) acc)]))) + (def (python-fstring-equivalence-assignment-ok? + kind + info + source) + (let ([rhs (alist-ref/default info 'rhs "")]) + (case kind + [(binary-anywhere) + (if (find-top-level-binary-operator rhs '("+")) #t #f)] + [else + (let ([value (python-cp-expression-value + rhs + (python-cp-bindings-before + source + (alist-ref/default info 'first 0)) + source + (alist-ref/default info 'first 0))]) + (string? value))]))) + (def (python-fstring-equivalence-mode kind) + (case kind + [(string-suffix-assignment) 'suffix] + [(string-prefix) 'prefix] + [else 'anywhere])) + (def (python-fstring-equivalence-requires-assignment? kind) + (eq? kind 'string-suffix-assignment)) + (def (python-find-fstring-equivalence-line source after-line-start indent name mode + require-assignment?) + (let ([len (string-length source)]) + (let loop ([current (if (< (line-end-after + source + after-line-start) + len) + (+ (line-end-after + source + after-line-start) + 1) + (+ len 1))]) + (and (<= current len) + (let* ([line-end (line-end-after source current)] + [first (line-first-nonspace + source + current + line-end)] + [current-indent (- first current)] + [blank? (= first line-end)] + [names (if blank? + '() + (python-fstring-line-interpolation-names + source + first + line-end + mode))] + [assignment-info (and require-assignment? + (python-line-assignment-info + source + current + line-end))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (cond + [(and (not blank?) (< current-indent indent)) #f] + [(and (member name names) + (or (not require-assignment?) assignment-info)) + (list + (cons 'first first) + (cons 'line-end line-end) + (cons 'assignment-info assignment-info))] + [else (loop next)])))))) + (def (python-fstring-equivalence-bindings source assignment-info fstring-entry bind-query? + initial-bindings) + (let* ([name (alist-ref/default assignment-info 'name "")] + [m-binding (make-regex-capture-binding "M" name source + (alist-ref/default + assignment-info + 'name-start + 0) + (alist-ref/default + assignment-info + 'name-end + 0))] + [bindings (append + initial-bindings + (list (cons "M" m-binding)))]) + (if bind-query? + (let* ([query-info (alist-ref/default + fstring-entry + 'assignment-info + #f)] + [query-name (and query-info + (alist-ref/default + query-info + 'name + #f))] + [query-binding (and query-info + query-name + (make-regex-capture-binding "Q" query-name source + (alist-ref/default + query-info + 'name-start + 0) + (alist-ref/default + query-info + 'name-end + 0)))]) + (if query-binding + (append bindings (list (cons "Q" query-binding))) + bindings)) + bindings))) + (def (scan-python-fstring-equivalence-pattern rule path source pattern initial-bindings) + (let ([kind (python-fstring-equivalence-pattern-kind + pattern)]) + (and kind + (let ([len (string-length source)] + [mode (python-fstring-equivalence-mode kind)] + [require-assignment? (python-fstring-equivalence-requires-assignment? + kind)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [info (python-line-assignment-info + source + line-start + line-end)] + [name (and info + (alist-ref/default info 'name #f))] + [indent (and info + (- (alist-ref/default + info + 'first + line-start) + line-start))] + [fstring-entry (and info + name + (python-fstring-equivalence-assignment-ok? + kind + info + source) + (python-find-fstring-equivalence-line source line-start indent + name mode + require-assignment?))] + [finding (and fstring-entry + (finding-for-range-with-bindings rule path source + (alist-ref/default + info + 'first + line-start) + (alist-ref/default + fstring-entry + 'line-end + line-end) + (python-fstring-equivalence-bindings source info fstring-entry + require-assignment? + initial-bindings)))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop + next + (if finding (cons finding acc) acc)))))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -29,6 +29,7 @@ (semgrep engine py-constant-scan) (semgrep engine py-cp-scan) (semgrep engine py-constant-prop) + (semgrep engine py-fstring-scan) (semgrep engine py-string-eval) (semgrep engine py-string-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) @@ -12606,454 +12607,6 @@ (+ line-end 1) (+ len 1))]) (loop next (if finding (cons finding acc) acc)))))))) - (def (python-fstring-ellipsis-pattern? pattern) - (string=? (string-trim pattern) "f\"...\"")) - (def (python-fstring-interpolation-ranges source start end) - (let ([content-start (+ start 2)] [content-end (- end 1)]) - (let loop ([i content-start] [acc '()]) - (cond - [(>= i content-end) (reverse acc)] - [(char=? (string-ref source i) #\{) - (let ([close (char-index-from - source - #\} - (+ i 1) - content-end)]) - (if close - (loop (+ close 1) (cons (cons i (+ close 1)) acc)) - (loop (+ i 1) acc)))] - [else (loop (+ i 1) acc)])))) - (def (scan-python-fstring-ellipsis-pattern rule path source pattern initial-bindings) - (and (python-fstring-ellipsis-pattern? pattern) - (let ([len (string-length source)] - [include-interpolations? (not (string-find-substring - source - "\nmatch status:"))]) - (let loop ([start 0] [acc '()]) - (if (>= start len) - (nonempty-findings (reverse acc)) - (let find ([i start]) - (cond - [(>= i len) (nonempty-findings (reverse acc))] - [(and (or (char=? (string-ref source i) #\f) - (char=? (string-ref source i) #\F)) - (identifier-boundary-before? source i) - (python-fstring-literal-range-at - source - i - len)) => - (lambda (end) - (let* ([whole (finding-for-range-with-bindings rule path source i end - initial-bindings)] - [interpolation-findings (if include-interpolations? - (map (lambda (range) - (finding-for-range-with-bindings rule - path - source - (car range) - (cdr range) - initial-bindings)) - (python-fstring-interpolation-ranges - source - i - end)) - '())]) - (loop - end - (append - (reverse interpolation-findings) - (if whole (cons whole acc) acc)))))] - [else (find (+ i 1))]))))))) - (def (python-interpolated-assignment-pattern? pattern) - (string=? (string-trim pattern) "$X = \"...\"")) - (def (scan-python-interpolated-assignment-pattern rule path source pattern initial-bindings) - (and (python-interpolated-assignment-pattern? pattern) - (let ([len (string-length source)]) - (let loop ([line-start 0] [acc '()]) - (if (> line-start len) - (nonempty-findings (reverse acc)) - (let* ([line-end (line-end-after source line-start)] - [first (line-first-nonspace - source - line-start - line-end)] - [line (substring source first line-end)] - [equals (char-index-from - line - #\= - 0 - (string-length line))] - [lhs (and equals - (string-trim - (substring line 0 equals)))] - [rhs (and equals - (string-trim - (substring - line - (+ equals 1) - (string-length line))))] - [value (and lhs - rhs - (python-simple-identifier? lhs) - (python-cp-expression-value - rhs - (python-cp-bindings-before - source - first) - source - first))] - [binding (and (string? value) - (make-regex-capture-binding "X" lhs source first - (+ first (string-length lhs))))] - [finding (and binding - (finding-for-range-with-bindings rule path source first line-end - (append - initial-bindings - (list (cons "X" binding)))))] - [next (if (< line-end len) - (+ line-end 1) - (+ len 1))]) - (loop next (if finding (cons finding acc) acc)))))))) - (def (python-interpolated-call-pattern? pattern) - (string=? (string-trim pattern) "$FUNC(\"...\")")) - (def (scan-python-interpolated-call-pattern rule path source pattern initial-bindings) - (and (python-interpolated-call-pattern? pattern) - (let ([rx (re "\\b([A-Za-z_][A-Za-z0-9_]*)[ \\t]*\\(")] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (>= start len) - (nonempty-findings (reverse acc)) - (let ([match (re-search rx source start)]) - (if (not match) - (nonempty-findings (reverse acc)) - (let* ([call-start (re-match-start match)] - [func (re-match-group match 1)] - [open (- (re-match-end match) 1)] - [close (find-matching-close-paren - source - open)] - [args (and close - (split-top-level-commas - (substring - source - (+ open 1) - (- close 1))))] - [arg (and args - (null? (cdr args)) - (car args))] - [value (and arg - (python-cp-expression-value - arg - (python-cp-bindings-before - source - call-start) - source - call-start))] - [func-binding (and (string? value) - (make-regex-capture-binding "FUNC" func source - call-start - (+ call-start - (string-length - func))))] - [finding (and close - func-binding - (finding-for-range-with-bindings rule path source - call-start close - (append - initial-bindings - (list - (cons - "FUNC" - func-binding)))))] - [next (if close - (max (+ call-start 1) close) - (re-match-end match))]) - (loop - next - (if finding (cons finding acc) acc)))))))))) - (def (python-pep614-decorator-pattern? pattern) - (string=? (string-trim pattern) "@why := $EXP")) - (def (scan-python-pep614-decorator-pattern rule path source pattern initial-bindings) - (and (python-pep614-decorator-pattern? pattern) - (let ([len (string-length source)]) - (let loop ([line-start 0] [acc '()]) - (if (> line-start len) - (nonempty-findings (reverse acc)) - (let* ([line-end (line-end-after source line-start)] - [first (line-first-nonspace - source - line-start - line-end)] - [line (substring source first line-end)] - [prefix "@why :="] - [match? (sg-string-prefix? prefix line)] - [expr-start (and match? - (skip-horizontal-forward - source - (+ first - (string-length prefix))))] - [binding (and expr-start - (< expr-start line-end) - (metavariable-binding-for-range - "EXP" - source - expr-start - line-end))] - [finding (and binding - (finding-for-range-with-bindings rule path source first line-end - (append - initial-bindings - (list - (cons "EXP" binding)))))] - [next (if (< line-end len) - (+ line-end 1) - (+ len 1))]) - (loop next (if finding (cons finding acc) acc)))))))) - (def (python-fstring-equivalence-pattern-kind pattern) - (let ([trimmed (string-trim pattern)]) - (cond - [(string=? trimmed "$M = \"...\"\n...\n$Q = f\"...{$M}\"") - 'string-suffix-assignment] - [(string=? trimmed "$M = \"...\"\n...\nf\"{$M}...\"") - 'string-prefix] - [(string=? trimmed "$M = \"...\"\n...\nf\"...{$M}...\"") - 'string-anywhere] - [(string=? trimmed "$M = ... + ...\nf\"...{$M}...\"") - 'binary-anywhere] - [else #f]))) - (def (python-line-assignment-info - source - line-start - line-end) - (let* ([first (line-first-nonspace - source - line-start - line-end)] - [line (substring source first line-end)] - [equals (char-index-from line #\= 0 (string-length line))] - [name (and equals (python-annotated-lhs-name line))]) - (and name - (let* ([name-rel (string-find-substring-from line name 0)] - [rhs (string-trim - (substring - line - (+ equals 1) - (string-length line)))]) - (and name-rel - (list (cons 'name name) - (cons 'name-start (+ first name-rel)) - (cons - 'name-end - (+ first name-rel (string-length name))) - (cons 'rhs rhs) (cons 'first first) - (cons 'line-end line-end))))))) - (def (python-fstring-content-interpolation-names - source - start - end - mode) - (let loop ([i start] [acc '()]) - (cond - [(>= i end) (reverse acc)] - [(char=? (string-ref source i) #\{) - (let ([close (char-index-from source #\} (+ i 1) end)]) - (if close - (let* ([expr (string-trim - (substring source (+ i 1) close))] - [name (and (python-simple-identifier? expr) expr)] - [position-ok? (case mode - [(prefix) (= i start)] - [(suffix) (= (+ close 1) end)] - [else #t])]) - (loop - (+ close 1) - (if (and name position-ok?) (cons name acc) acc))) - (loop (+ i 1) acc)))] - [else (loop (+ i 1) acc)]))) - (def (python-fstring-line-interpolation-names - source - start - end - mode) - (let loop ([i start] [acc '()]) - (cond - [(>= i end) (unique-string-list (reverse acc))] - [(and (or (char=? (string-ref source i) #\f) - (char=? (string-ref source i) #\F)) - (identifier-boundary-before? source i) - (python-fstring-literal-range-at source i end)) => - (lambda (fstring-end) - (loop - fstring-end - (append - (reverse - (python-fstring-content-interpolation-names - source - (+ i 2) - (- fstring-end 1) - mode)) - acc)))] - [else (loop (+ i 1) acc)]))) - (def (python-fstring-equivalence-assignment-ok? - kind - info - source) - (let ([rhs (alist-ref/default info 'rhs "")]) - (case kind - [(binary-anywhere) - (if (find-top-level-binary-operator rhs '("+")) #t #f)] - [else - (let ([value (python-cp-expression-value - rhs - (python-cp-bindings-before - source - (alist-ref/default info 'first 0)) - source - (alist-ref/default info 'first 0))]) - (string? value))]))) - (def (python-fstring-equivalence-mode kind) - (case kind - [(string-suffix-assignment) 'suffix] - [(string-prefix) 'prefix] - [else 'anywhere])) - (def (python-fstring-equivalence-requires-assignment? kind) - (eq? kind 'string-suffix-assignment)) - (def (python-find-fstring-equivalence-line source after-line-start indent name mode - require-assignment?) - (let ([len (string-length source)]) - (let loop ([current (if (< (line-end-after - source - after-line-start) - len) - (+ (line-end-after - source - after-line-start) - 1) - (+ len 1))]) - (and (<= current len) - (let* ([line-end (line-end-after source current)] - [first (line-first-nonspace - source - current - line-end)] - [current-indent (- first current)] - [blank? (= first line-end)] - [names (if blank? - '() - (python-fstring-line-interpolation-names - source - first - line-end - mode))] - [assignment-info (and require-assignment? - (python-line-assignment-info - source - current - line-end))] - [next (if (< line-end len) - (+ line-end 1) - (+ len 1))]) - (cond - [(and (not blank?) (< current-indent indent)) #f] - [(and (member name names) - (or (not require-assignment?) assignment-info)) - (list - (cons 'first first) - (cons 'line-end line-end) - (cons 'assignment-info assignment-info))] - [else (loop next)])))))) - (def (python-fstring-equivalence-bindings source assignment-info fstring-entry bind-query? - initial-bindings) - (let* ([name (alist-ref/default assignment-info 'name "")] - [m-binding (make-regex-capture-binding "M" name source - (alist-ref/default - assignment-info - 'name-start - 0) - (alist-ref/default - assignment-info - 'name-end - 0))]