Extract Semgrep Python import scanners
ober
8f2c6fad581eedfb367e1fb4858629c8707001d5
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -505,6 +505,9 @@ Completed in the repo: `src/semgrep/engine/py-fstring-scan.ss` - extracted Python `return "..."` and `[...]` fallback scanners into `src/semgrep/engine/py-return-scan.ss` + - extracted Python import-local, import-equivalence, imported dotted-call, + and star-import-qualified fallback scanners into + `src/semgrep/engine/py-import-scan.ss` - exported shared Python line-assignment analysis from `src/semgrep/engine/py-constant-prop.ss` so remaining Python fallbacks in `scan.ss` can reuse one assignment-info implementation @@ -524,10 +527,10 @@ Remaining Phase 0 work: - Continue splitting `src/semgrep/scan.ss` into engine, targeting, text-mode, structural dispatch, taint, and result/output-adjacent modules. The Python - constant-propagation and f-string/interpolated fallback surfaces are now - out, and the remaining Python return/list special cases are out too; the - next meaningful cuts are the Python import-local/equivalence special-case - scanners plus the larger language dispatch and taint sections. + constant-propagation, f-string/interpolated, return/list, and import + special-case fallback surfaces are now out; the next meaningful cuts are the + larger language dispatch, import-map/shared textual helper ownership, and + taint sections. - Run broader upstream sweeps and grow the expected-fail baselines from real sampled data instead of one narrow seed run. new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-import-scan.sls @@ -0,0 +1,706 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-import-scan.ss + +(library (semgrep engine py-import-scan) + (export trim-python-module-finding scan-python-import-local-pattern + scan-python-import-local-pattern-with-bindings + scan-python-import-equivalence-pattern + scan-python-import-equivalent-call-pattern + scan-python-star-import-qualified-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (semgrep result) (semgrep result findings) + (semgrep engine regex-support) + (semgrep engine text-support)) + (def (sg-string-prefix? prefix s) + (let ([prefix-len (string-length prefix)] + [len (string-length s)]) + (and (<= prefix-len len) + (string=? (substring s 0 prefix-len) prefix)))) + (def (sg-string-suffix? suffix s) + (let ([suffix-len (string-length suffix)] + [len (string-length s)]) + (and (<= suffix-len len) + (string=? (substring s (- len suffix-len) len) suffix)))) + (def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + (def (alist-ref/default xs key default) + (let ([found (assoc key xs)]) + (if found (cdr found) default))) + (def (source-content-end source) + (let loop ([i (string-length source)]) + (if (and (> i 0) + (char=? (string-ref source (- i 1)) #\newline)) + (loop (- i 1)) + i))) + (def (symbolic-python-like-language? language) + (or (string=? language "python") + (string=? language "python2") + (string=? language "python3") + (string=? language "py"))) + (def (unique-string-list xs) + (let loop ([remaining xs] [seen '()] [acc '()]) + (cond + [(null? remaining) (reverse acc)] + [(or (not (car remaining)) (member (car remaining) seen)) + (loop (cdr remaining) seen acc)] + [else + (loop + (cdr remaining) + (cons (car remaining) seen) + (cons (car remaining) acc))]))) + (def (any? pred xs) + (and (not (null? xs)) + (or (pred (car xs)) (any? pred (cdr xs))))) + (def (all? pred xs) + (or (null? xs) (and (pred (car xs)) (all? pred (cdr xs))))) + (def (identifier-token-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_))) + (def (token-start-boundary? source index) + (or (= index 0) + (not (identifier-token-char? + (string-ref source (- index 1)))))) + (def (token-end-boundary? source index) + (or (= index (string-length source)) + (not (identifier-token-char? (string-ref source index))))) + (def (skip-horizontal-backward source i) + (let loop ([j i]) + (if (and (>= j 0) + (let ([ch (string-ref source j)]) + (or (char=? ch #\space) (char=? ch #\tab)))) + (loop (- j 1)) + j))) + (def (replace-range text start end replacement) + (string-append + (substring text 0 start) + replacement + (substring text end (string-length text)))) + (def (find-matching-close-paren source open-index) + (let ([len (string-length source)]) + (let loop ([i open-index] + [depth 0] + [state 'normal] + [escaped? #f]) + (cond + [(>= i len) #f] + [(eq? state 'normal) + (let ([ch (string-ref source i)]) + (cond + [(char=? ch #\") (loop (+ i 1) depth 'double #f)] + [(char=? ch #\') (loop (+ i 1) depth 'single #f)] + [(char=? ch #\`) (loop (+ i 1) depth 'backtick #f)] + [(char=? ch #\() (loop (+ i 1) (+ depth 1) state #f)] + [(char=? ch #\)) + (if (= depth 1) + (+ i 1) + (loop (+ i 1) (max 0 (- depth 1)) state #f))] + [else (loop (+ i 1) depth state #f)]))] + [escaped? (loop (+ i 1) depth state #f)] + [(char=? (string-ref source i) #\\) + (loop (+ i 1) depth state #t)] + [(and (eq? state 'double) + (char=? (string-ref source i) #\")) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'single) + (char=? (string-ref source i) #\')) + (loop (+ i 1) depth 'normal #f)] + [(and (eq? state 'backtick) + (char=? (string-ref source i) #\`)) + (loop (+ i 1) depth 'normal #f)] + [else (loop (+ i 1) depth state #f)])))) + (def (dotted-name-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$) + (char=? ch #\.))) + (def (dotted-name-pattern? text) + (let ([len (string-length text)]) + (and (> len 0) + (let loop ([i 0]) + (cond + [(= i len) #t] + [(char=? (string-ref text i) #\.) + (and (> i 0) (< (+ i 1) len) (loop (+ i 1)))] + [(dotted-name-char? (string-ref text i)) (loop (+ i 1))] + [else #f]))))) + (def (dotted-name? text) + (and (dotted-name-pattern? text) + (string-find-substring text "."))) + (def (regex-fold-matches pattern source proc seed) + (let ([rx (re pattern)] [len (string-length source)]) + (let loop ([start 0] [acc seed]) + (if (> start len) + acc + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next (proc match acc))) + acc)))))) + (def (add-name-import local fqn imports) + (let ([found (assoc local imports)]) + (cond + [(not found) (cons (cons local (list fqn)) imports)] + [(member fqn (cdr found)) imports] + [else + (cons + (cons local (cons fqn (cdr found))) + (let loop ([xs imports]) + (cond + [(null? xs) '()] + [(eq? found (car xs)) (loop (cdr xs))] + [else (cons (car xs) (loop (cdr xs)))])))]))) + (def (split-on-char source delimiter) + (let ([len (string-length source)]) + (let loop ([i 0] [start 0] [acc '()]) + (cond + [(= i len) + (reverse + (cons (string-trim (substring source start len)) acc))] + [(char=? (string-ref source i) delimiter) + (loop + (+ i 1) + (+ i 1) + (cons (string-trim (substring source start i)) acc))] + [else (loop (+ i 1) start acc)])))) + (def (first-index-of source ch) + (let ([len (string-length source)]) + (let loop ([i 0]) + (cond + [(= i len) #f] + [(char=? (string-ref source i) ch) i] + [else (loop (+ i 1))])))) + (def (split-alias item) + (let* ([trimmed (string-trim item)] + [as-index (string-find-substring trimmed " as ")]) + (if as-index + (values + (string-trim (substring trimmed 0 as-index)) + (string-trim + (substring + trimmed + (+ as-index 4) + (string-length trimmed)))) + (values trimmed #f)))) + (def (first-dotted-segment name) + (let ([dot (first-index-of name #\.)]) + (if dot (substring name 0 dot) name))) + (def (parse-python-import-items items imports) + (let loop ([xs (split-on-char items #\,)] [acc imports]) + (if (null? xs) + acc + (let-values ([(imported alias) (split-alias (car xs))]) + (loop + (cdr xs) + (add-name-import + (or alias (first-dotted-segment imported)) + imported + acc)))))) + (def (parse-python-from-import-items module items imports) + (let loop ([xs (split-on-char items #\,)] [acc imports]) + (if (null? xs) + acc + (let-values ([(imported alias) (split-alias (car xs))]) + (if (or (string=? imported "*") (string=? imported "")) + (loop (cdr xs) acc) + (loop + (cdr xs) + (add-name-import + (or alias imported) + (string-append module "." imported) + acc))))))) + (def (python-import-map source) + (let* ([from-imports (regex-fold-matches + "(^|\\n)\\s*from\\s+([A-Za-z_][A-Za-z0-9_\\.]*?)\\s+import\\s+([^\\n#]+)" + source + (lambda (match acc) + (parse-python-from-import-items + (re-match-group match 2) + (re-match-group match 3) + acc)) + '())] + [plain-imports (regex-fold-matches + "(^|\\n)\\s*import\\s+([^\\n#]+)" + source + (lambda (match acc) + (parse-python-import-items + (re-match-group match 2) + acc)) + from-imports)]) + plain-imports)) + (def (python-leading-code-offset source) + (let ([len (string-length source)]) + (let loop ([line-start 0]) + (if (>= line-start len) + len + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [blank? (= first line-end)] + [comment? (and (< first line-end) + (char=? + (string-ref source first) + #\#))] + [next (if (< line-end len) (+ line-end 1) len)]) + (if (or blank? comment?) (loop next) first)))))) + (def (trim-python-module-finding finding source) + (let* ([start (python-leading-code-offset source)] + [end (source-content-end source)]) + (if (and (= (finding-start-offset finding) 0) + (>= (finding-end-offset finding) end) + (< start end)) + (finding-with-range finding source start end) + finding))) + (def (replace-imported-fqn-pattern pattern local fqn) + (let ([fqn-len (string-length fqn)]) + (let loop ([start 0]) + (let ([index (string-find-substring-from + pattern + fqn + start)]) + (and index + (let ([end (+ index fqn-len)]) + (if (and (token-start-boundary? pattern index) + (token-end-boundary? pattern end)) + (replace-range pattern index end local) + (loop (+ index 1))))))))) + (def (python-import-local-pattern-candidates source pattern) + (unique-string-list + (let import-loop ([imports (python-import-map source)] + [acc '()]) + (if (null? imports) + acc + (let* ([entry (car imports)] + [local (car entry)] + [fqns (cdr entry)]) + (import-loop + (cdr imports) + (let fqn-loop ([xs fqns] [inner acc]) + (if (null? xs) + inner + (fqn-loop + (cdr xs) + (cons + (replace-imported-fqn-pattern + pattern + local + (car xs)) + inner)))))))))) + (def (scan-python-import-local-pattern rule language path source target-root pattern + structural-scan) + (and (symbolic-python-like-language? language) + (let ([patterns (python-import-local-pattern-candidates + source + pattern)]) + (and (not (null? patterns)) + (let ([findings (apply + append + (map (lambda (candidate) + (structural-scan rule language path source + target-root candidate)) + patterns))]) + (and (not (null? findings)) + (map (lambda (finding) + (trim-python-module-finding + finding + source)) + findings))))))) + (def (scan-python-import-local-pattern-with-bindings rule language path source target-root pattern + initial-bindings structural-scan-with-bindings) + (and (symbolic-python-like-language? language) + (let ([patterns (python-import-local-pattern-candidates + source + pattern)]) + (and (not (null? patterns)) + (let ([findings (apply + append + (map (lambda (candidate) + (structural-scan-with-bindings rule language path source + target-root candidate + initial-bindings)) + patterns))]) + (and (not (null? findings)) + (map (lambda (finding) + (trim-python-module-finding + finding + source)) + findings))))))) + (def (python-import-clean-item item) + (let ([without-delimiters (let ([len (string-length item)]) + (let loop ([i 0] [acc '()]) + (cond + [(= i len) + (list->string (reverse acc))] + [(let ([ch (string-ref item i)]) + (or (char=? ch #\() + (char=? ch #\)))) + (loop (+ i 1) acc)] + [else + (loop + (+ i 1) + (cons + (string-ref item i) + acc))])))]) + (string-trim without-delimiters))) + (def (python-import-entry kind fqn) (cons kind fqn)) + (def (python-import-entry-kind entry) (car entry)) + (def (python-import-entry-fqn entry) (cdr entry)) + (def (python-import-items->entries items module) + (let loop ([xs (split-on-char items #\,)] [acc '()]) + (if (null? xs) + (reverse acc) + (let* ([clean (python-import-clean-item (car xs))]) + (let-values ([(imported alias) (split-alias clean)]) + (cond + [(string=? imported "") (loop (cdr xs) acc)] + [(string=? imported "*") + (loop + (cdr xs) + (if module + (cons (python-import-entry 'star module) acc) + acc))] + [else + (loop + (cdr xs) + (cons + (python-import-entry + 'normal + (if module + (string-append module "." imported) + imported)) + acc))])))))) + (def (python-import-line-without-comment line) + (let ([comment (first-index-of line #\#)]) + (if comment (substring line 0 comment) line))) + (def (python-absolute-module-name? module) + (and (> (string-length module) 0) + (not (char=? (string-ref module 0) #\.)))) + (def (python-import-line-entries line) + (let* ([trimmed (string-trim + (python-import-line-without-comment line))] + [len (string-length trimmed)]) + (cond + [(sg-string-prefix? "import " trimmed) + (python-import-items->entries (substring trimmed 7 len) #f)] + [(sg-string-prefix? "from " trimmed) + (let ([import-index (string-find-substring + trimmed + " import ")]) + (if import-index + (let ([module (string-trim + (substring trimmed 5 import-index))]) + (if (python-absolute-module-name? module) + (python-import-items->entries + (substring trimmed (+ import-index 8) len) + module) + '())) + '()))] + [else '()]))) + (def (python-import-pattern-requirements pattern) + (let ([trimmed (string-trim pattern)]) + (and (not (string-find-substring trimmed "\n")) + (not (string-find-substring trimmed "$")) + (not (string-find-substring trimmed "...")) + (let ([entries (python-import-line-entries trimmed)]) + (and (not (null? entries)) + (map python-import-entry-fqn entries)))))) + (def (python-import-entry-satisfies? entry required) + (let ([fqn (python-import-entry-fqn entry)]) + (if (eq? (python-import-entry-kind entry) 'star) + (string=? fqn required) + (or (string=? fqn required) + (sg-string-prefix? (string-append required ".") fqn))))) + (def (python-import-line-satisfies? entries requirements) + (all? + (lambda (required) + (any? + (lambda (entry) + (python-import-entry-satisfies? entry required)) + entries)) + requirements)) + (def (python-import-trimmed-end source start end) + (let ([last (skip-horizontal-backward source (- end 1))]) + (cond + [(< last start) start] + [(char=? (string-ref source last) #\)) last] + [else (+ last 1)]))) + (def (python-import-item-matches-requirement? + imported + requirements) + (let ([entry (python-import-entry 'normal imported)]) + (any? + (lambda (required) + (python-import-entry-satisfies? entry required)) + requirements))) + (def (python-import-line-matching-import-end + line + requirements) + (let ([len (string-length line)]) + (let loop ([start 7] [i 7] [best #f]) + (cond + [(> start len) best] + [(or (= i len) (char=? (string-ref line i) #\,)) + (let* ([item-end (python-import-trimmed-end line start i)] + [item (substring line start item-end)]) + (let-values ([(imported alias) + (split-alias + (python-import-clean-item item))]) + (loop + (+ i 1) + (+ i 1) + (if (and (not (string=? imported "")) + (python-import-item-matches-requirement? + imported + requirements)) + item-end + best))))] + [else (loop start (+ i 1) best)])))) + (def (python-import-line-match-end + source + first + line-end + requirements) + (let* ([line (substring source first line-end)] + [trimmed-line (string-trim line)]) + (if (sg-string-prefix? "import " trimmed-line) + (let ([relative-end (python-import-line-matching-import-end + trimmed-line + requirements)]) + (if relative-end + (+ first relative-end) + (python-import-trimmed-end source first line-end))) + (python-import-trimmed-end source first line-end)))) + (def (scan-python-import-equivalence-pattern rule path source pattern initial-bindings) + (let ([requirements (python-import-pattern-requirements + pattern)]) + (and requirements + (let ([len (string-length source)]) + (let loop ([line-start 0] [acc '()]) + (if (> line-start len) + (nonempty-findings (reverse acc)) + (let* ([line-end (line-end-after source line-start)] + [first (line-first-nonspace + source + line-start + line-end)] + [line (substring source first line-end)] + [entries (python-import-line-entries line)] + [finding (and (not (null? entries)) + (python-import-line-satisfies? + entries + requirements) + (finding-for-range-with-bindings rule path source first + (python-import-line-match-end + source + first + line-end + requirements) + initial-bindings))] + [next (if (< line-end len) + (+ line-end 1) + (+ len 1))]) + (loop + next + (if finding (cons finding acc) acc))))))))) + (def (python-dotted-call-ellipsis-callee pattern) + (let* ([trimmed (string-trim pattern)] + [len (string-length trimmed)] + [suffix "(...)"]) + (and (> len (string-length suffix)) + (sg-string-suffix? suffix trimmed) + (let ([callee (substring + trimmed + 0 + (- len (string-length suffix)))]) + (and (dotted-name-pattern? callee) + (dotted-name? callee) + callee))))) + (def (python-import-local-callee-candidates source callee) + (unique-string-list + (let import-loop ([imports (python-import-map source)] + [acc (list callee)]) + (if (null? imports) + acc + (let* ([entry (car imports)] + [local (car entry)] + [fqns (cdr entry)]) + (import-loop + (cdr imports) + (let fqn-loop ([xs fqns] [inner acc]) + (if (null? xs) + inner + (let* ([fqn (car xs)] + [candidate (cond + [(and (string=? fqn callee) + (not (string=? + local + (first-dotted-segment + fqn)))) + local] + [(sg-string-prefix? + (string-append fqn ".") + callee) + (string-append + local + (substring + callee + (string-length fqn) + (string-length callee)))] + [else #f])]) + (fqn-loop + (cdr xs) + (if candidate + (cons candidate inner) + inner))))))))))) + (def (dotted-token-start-boundary? source index) + (or (= index 0) + (let ([ch (string-ref source (- index 1))]) + (and (not (identifier-token-char? ch)) + (not (char=? ch #\.)))))) + (def (dotted-token-end-boundary? source index) + (or (>= index (string-length source)) + (let ([ch (string-ref source index)]) + (and (not (identifier-token-char? ch)) + (not (char=? ch #\.)))))) + (def (scan-python-callee-occurrences rule path source callee + initial-bindings) + (let ([len (string-length source)] + [callee-len (string-length callee)]) + (let loop ([start 0] [acc '()]) + (let ([index (string-find-substring-from + source + callee + start)]) + (if (not index) + (reverse acc) + (let* ([after-callee (+ index callee-len)] + [open (skip-horizontal-forward + source + after-callee)] + [close (and (< open len) + (char=? (string-ref source open) #\() + (find-matching-close-paren + source + open))] + [finding (and close + (dotted-token-start-boundary? + source + index) + (finding-for-range-with-bindings rule path source index close + initial-bindings))] + [next (max (+ index 1) + (if close close after-callee))]) + (loop next (if finding (cons finding acc) acc)))))))) + (def (insert-finding-by-start finding sorted) + (cond + [(null? sorted) (list finding)] + [(< (finding-start-offset finding) + (finding-start-offset (car sorted))) + (cons finding sorted)] + [else + (cons + (car sorted) + (insert-finding-by-start finding (cdr sorted)))])) + (def (sort-findings-by-start findings) + (let loop ([xs findings] [acc '()]) + (if (null? xs) + acc + (loop (cdr xs) (insert-finding-by-start (car xs) acc))))) + (def (scan-python-import-equivalent-call-pattern rule path source pattern initial-bindings) + (let ([callee (python-dotted-call-ellipsis-callee pattern)]) + (and callee + (let ([findings (apply + append + (map (lambda (candidate) + (scan-python-callee-occurrences rule path source candidate + initial-bindings)) + (python-import-local-callee-candidates + source + callee)))]) + (nonempty-findings + (sort-findings-by-start (dedupe-findings findings))))))) + (def (python-star-import-modules source) + (regex-fold-matches + "(^|\\n)from[ \\t]+([A-Za-z_][A-Za-z0-9_\\.]*)[ \\t]+import[ \\t]+\\*" + source + (lambda (match acc) + (let ([module (re-match-group match 2)]) + (if (member module acc) acc (cons module acc)))) + '())) + (def (python-star-import-tail-candidates source pattern) + (let ([trimmed (string-trim pattern)]) + (and (dotted-name-pattern? trimmed) + (let loop ([modules (python-star-import-modules source)] + [acc '()]) + (if (null? modules) + (unique-string-list (reverse acc)) + (let* ([module (car modules)] + [prefix (string-append module ".")] + [tail (and (sg-string-prefix? prefix trimmed) + (substring + trimmed + (string-length prefix) + (string-length trimmed)))]) + (loop + (cdr modules) + (if (and tail + (> (string-length tail) 0) + (dotted-name-pattern? tail)) + (cons tail acc) + acc)))))))) + (def (scan-python-dotted-token-occurrences rule path source + token initial-bindings) + (let ([token-len (string-length token)]) + (let loop ([start 0] [acc '()]) + (let ([index (string-find-substring-from + source + token + start)]) + (if (not index) + (reverse acc) + (let* ([end (+ index token-len)] + [finding (and (dotted-token-start-boundary? + source + index) + (dotted-token-end-boundary? + source + end) + (finding-for-range-with-bindings rule path source index end + initial-bindings))]) + (loop + (max (+ index 1) end) + (if finding (cons finding acc) acc)))))))) + (def (scan-python-star-import-qualified-pattern rule path source pattern initial-bindings) + (let ([tails (python-star-import-tail-candidates + source + pattern)]) + (and tails + (not (null? tails)) + (let* ([tokens (unique-string-list + (cons (string-trim pattern) tails))] + [findings (apply + append + (map (lambda (token) + (scan-python-dotted-token-occurrences rule path source token + initial-bindings)) + tokens))]) + (nonempty-findings + (sort-findings-by-start (dedupe-findings findings)))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -30,6 +30,7 @@ (semgrep engine py-cp-scan) (semgrep engine py-constant-prop) (semgrep engine py-fstring-scan) + (semgrep engine py-import-scan) (semgrep engine py-return-scan) (semgrep engine py-string-eval) (semgrep engine py-string-scan) (semgrep engine regex-scan) @@ -8098,105 +8099,6 @@ (string=? language "python2") (string=? language "python3") (string=? language "py"))) - (def (replace-imported-fqn-pattern pattern local fqn) - (let ([fqn-len (string-length fqn)]) - (let loop ([start 0]) - (let ([index (string-find-substring-from - pattern - fqn - start)]) - (and index - (let ([end (+ index fqn-len)]) - (if (and (token-start-boundary? pattern index) - (token-end-boundary? pattern end)) - (replace-range pattern index end local) - (loop (+ index 1))))))))) - (def (python-import-local-pattern-candidates source pattern) - (unique-string-list - (let import-loop ([imports (python-import-map source)] - [acc '()]) - (if (null? imports) - acc - (let* ([entry (car imports)] - [local (car entry)] - [fqns (cdr entry)]) - (import-loop - (cdr imports) - (let fqn-loop ([xs fqns] [inner acc]) - (if (null? xs) - inner - (fqn-loop - (cdr xs) - (cons - (replace-imported-fqn-pattern - pattern - local - (car xs)) - inner)))))))))) - (def (scan-python-import-local-pattern rule language path - source target-root pattern) - (and (symbolic-python-like-language? language) - (let ([patterns (python-import-local-pattern-candidates - source - pattern)]) - (and (not (null? patterns)) - (let ([findings (apply - append - (map (lambda (candidate) - (scan-structural-pattern rule language path source - target-root candidate)) - patterns))]) - (and (not (null? findings)) - (map (lambda (finding) - (trim-python-module-finding - finding - source)) - findings))))))) - (def (scan-python-import-local-pattern-with-bindings rule language path source target-root pattern - initial-bindings) - (and (symbolic-python-like-language? language) - (let ([patterns (python-import-local-pattern-candidates - source - pattern)]) - (and (not (null? patterns)) - (let ([findings (apply - append - (map (lambda (candidate) - (scan-structural-pattern-with-bindings rule language path source - target-root candidate - initial-bindings)) - patterns))]) - (and (not (null? findings)) - (map (lambda (finding) - (trim-python-module-finding - finding - source)) - findings))))))) - (def (python-leading-code-offset source) - (let ([len (string-length source)]) - (let loop ([line-start 0]) - (if (>= line-start len) - len - (let* ([line-end (line-end-after source line-start)] - [first (line-first-nonspace - source - line-start - line-end)] - [blank? (= first line-end)] - [comment? (and (< first line-end) - (char=? - (string-ref source first) - #\#))] - [next (if (< line-end len) (+ line-end 1) len)]) - (if (or blank? comment?) (loop next) first)))))) - (def (trim-python-module-finding finding source) - (let* ([start (python-leading-code-offset source)] - [end (source-content-end source)]) - (if (and (= (finding-start-offset finding) 0) - (>= (finding-end-offset finding) end) - (< start end)) - (finding-with-range finding source start end) - finding))) (def (find-matching-close-bracket source open-index) (let ([len (string-length source)]) (let loop ([i open-index] @@ -12643,127 +12545,6 @@ (+ first relative-end) (python-import-trimmed-end source first line-end))) (python-import-trimmed-end source first line-end)))) - (def (scan-python-import-equivalence-pattern rule path source pattern initial-bindings) - (let ([requirements (python-import-pattern-requirements - pattern)]) - (and requirements - (let ([len (string-length source)]) - (let loop ([line-start 0] [acc '()]) - (if (> line-start len) - (nonempty-findings (reverse acc)) - (let* ([line-end (line-end-after source line-start)] - [first (line-first-nonspace - source - line-start - line-end)] - [line (substring source first line-end)] - [entries (python-import-line-entries line)] - [finding (and (not (null? entries)) - (python-import-line-satisfies? - entries - requirements) - (finding-for-range-with-bindings rule path source first - (python-import-line-match-end - source - first - line-end - requirements) - initial-bindings))] - [next (if (< line-end len) - (+ line-end 1) - (+ len 1))]) - (loop - next - (if finding (cons finding acc) acc))))))))) - (def (python-dotted-call-ellipsis-callee pattern) - (let* ([trimmed (string-trim pattern)] - [len (string-length trimmed)] - [suffix "(...)"]) - (and (> len (string-length suffix)) - (sg-string-suffix? suffix trimmed) - (let ([callee (substring - trimmed - 0 - (- len (string-length suffix)))]) - (and (dotted-name-pattern? callee) - (dotted-name? callee) - callee))))) - (def (python-import-local-callee-candidates source callee) - (unique-string-list - (let import-loop ([imports (python-import-map source)] - [acc (list callee)]) - (if (null? imports) - acc - (let* ([entry (car imports)] - [local (car entry)] - [fqns (cdr entry)]) - (import-loop - (cdr imports) - (let fqn-loop ([xs fqns] [inner acc]) - (if (null? xs) - inner - (let* ([fqn (car xs)] - [candidate (cond - [(and (string=? fqn callee) - (not (string=? - local - (first-dotted-segment - fqn)))) - local] - [(sg-string-prefix? - (string-append fqn ".") - callee) - (string-append - local - (substring - callee - (string-length fqn) - (string-length callee)))] - [else #f])]) - (fqn-loop - (cdr xs) - (if candidate - (cons candidate inner) - inner))))))))))) - (def (dotted-token-start-boundary? source index) - (or (= index 0) - (let ([ch (string-ref source (- index 1))]) - (and (not (identifier-token-char? ch)) - (not (char=? ch #\.)))))) - (def (dotted-token-end-boundary? source index) - (or (>= index (string-length source)) - (let ([ch (string-ref source index)]) - (and (not (identifier-token-char? ch)) - (not (char=? ch #\.)))))) - (def (scan-python-callee-occurrences rule path source callee - initial-bindings) - (let ([len (string-length source)] - [callee-len (string-length callee)]) - (let loop ([start 0] [acc '()]) - (let ([index (string-find-substring-from - source - callee - start)]) - (if (not index) - (reverse acc) - (let* ([after-callee (+ index callee-len)] - [open (skip-horizontal-forward - source - after-callee)] - [close (and (< open len) - (char=? (string-ref source open) #\() - (find-matching-close-paren - source - open))] - [finding (and close - (dotted-token-start-boundary? - source - index) - (finding-for-range-with-bindings rule path source index close - initial-bindings))] - [next (max (+ index 1) - (if close close after-callee))]) - (loop next (if finding (cons finding acc) acc)))))))) (def (insert-finding-by-start finding sorted) (cond [(null? sorted) (list finding)] @@ -12779,86 +12560,6 @@ (if (null? xs) acc (loop (cdr xs) (insert-finding-by-start (car xs) acc))))) - (def (scan-python-import-equivalent-call-pattern rule path source pattern initial-bindings) - (let ([callee (python-dotted-call-ellipsis-callee pattern)]) - (and callee - (let ([findings (apply - append - (map (lambda (candidate) - (scan-python-callee-occurrences rule path source candidate - initial-bindings)) - (python-import-local-callee-candidates - source - callee)))]) - (nonempty-findings - (sort-findings-by-start (dedupe-findings findings))))))) - (def (python-star-import-modules source) - (regex-fold-matches - "(^|\\n)from[ \\t]+([A-Za-z_][A-Za-z0-9_\\.]*)[ \\t]+import[ \\t]+\\*" - source - (lambda (match acc)