Deduplicate Semgrep Python import helpers
ober
b58e2158eb7d85d16a1682ae72dc23c0a3cef51e
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -508,6 +508,9 @@ Completed in the repo: - extracted Python import-local, import-equivalence, imported dotted-call, and star-import-qualified fallback scanners into `src/semgrep/engine/py-import-scan.ss` + - moved the shared Python import-map and import-line parsing helpers under + `src/semgrep/engine/py-import-scan.ss` so `scan.ss` no longer carries a + duplicate import parser implementation - exported shared Python line-assignment analysis from `src/semgrep/engine/py-constant-prop.ss` so remaining Python fallbacks in `scan.ss` can reuse one assignment-info implementation --- a/lib/semgrep/engine/py-import-scan.sls +++ b/lib/semgrep/engine/py-import-scan.sls @@ -3,7 +3,8 @@ ;;; Source: src/semgrep/engine/py-import-scan.ss (library (semgrep engine py-import-scan) - (export trim-python-module-finding scan-python-import-local-pattern + (export python-import-map python-import-clean-item + trim-python-module-finding scan-python-import-local-pattern scan-python-import-local-pattern-with-bindings scan-python-import-equivalence-pattern scan-python-import-equivalent-call-pattern --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -12400,151 +12400,6 @@ (+ line-end 1) (+ len 1))]) (loop next (if finding (cons finding acc) acc)))))))) - (def (python-import-clean-item item) - (let ([without-delimiters (let ([len (string-length item)]) - (let loop ([i 0] [acc '()]) - (cond - [(= i len) - (list->string (reverse acc))] - [(let ([ch (string-ref item i)]) - (or (char=? ch #\() - (char=? ch #\)))) - (loop (+ i 1) acc)] - [else - (loop - (+ i 1) - (cons - (string-ref item i) - acc))])))]) - (string-trim without-delimiters))) - (def (python-import-entry kind fqn) (cons kind fqn)) - (def (python-import-entry-kind entry) (car entry)) - (def (python-import-entry-fqn entry) (cdr entry)) - (def (python-import-items->entries items module) - (let loop ([xs (split-on-char items #\,)] [acc '()]) - (if (null? xs) - (reverse acc) - (let* ([clean (python-import-clean-item (car xs))]) - (let-values ([(imported alias) (split-alias clean)]) - (cond - [(string=? imported "") (loop (cdr xs) acc)] - [(string=? imported "*") - (loop - (cdr xs) - (if module - (cons (python-import-entry 'star module) acc) - acc))] - [else - (loop - (cdr xs) - (cons - (python-import-entry - 'normal - (if module - (string-append module "." imported) - imported)) - acc))])))))) - (def (python-import-line-without-comment line) - (let ([comment (first-index-of line #\#)]) - (if comment (substring line 0 comment) line))) - (def (python-absolute-module-name? module) - (and (> (string-length module) 0) - (not (char=? (string-ref module 0) #\.)))) - (def (python-import-line-entries line) - (let* ([trimmed (string-trim - (python-import-line-without-comment line))] - [len (string-length trimmed)]) - (cond - [(sg-string-prefix? "import " trimmed) - (python-import-items->entries (substring trimmed 7 len) #f)] - [(sg-string-prefix? "from " trimmed) - (let ([import-index (string-find-substring - trimmed - " import ")]) - (if import-index - (let ([module (string-trim - (substring trimmed 5 import-index))]) - (if (python-absolute-module-name? module) - (python-import-items->entries - (substring trimmed (+ import-index 8) len) - module) - '())) - '()))] - [else '()]))) - (def (python-import-pattern-requirements pattern) - (let ([trimmed (string-trim pattern)]) - (and (not (string-find-substring trimmed "\n")) - (not (string-find-substring trimmed "$")) - (not (string-find-substring trimmed "...")) - (let ([entries (python-import-line-entries trimmed)]) - (and (not (null? entries)) - (map python-import-entry-fqn entries)))))) - (def (python-import-entry-satisfies? entry required) - (let ([fqn (python-import-entry-fqn entry)]) - (if (eq? (python-import-entry-kind entry) 'star) - (string=? fqn required) - (or (string=? fqn required) - (sg-string-prefix? (string-append required ".") fqn))))) - (def (python-import-line-satisfies? entries requirements) - (all? - (lambda (required) - (any? - (lambda (entry) - (python-import-entry-satisfies? entry required)) - entries)) - requirements)) - (def (python-import-trimmed-end source start end) - (let ([last (skip-horizontal-backward source (- end 1))]) - (cond - [(< last start) start] - [(char=? (string-ref source last) #\)) last] - [else (+ last 1)]))) - (def (python-import-item-matches-requirement? - imported - requirements) - (let ([entry (python-import-entry 'normal imported)]) - (any? - (lambda (required) - (python-import-entry-satisfies? entry required)) - requirements))) - (def (python-import-line-matching-import-end - line - requirements) - (let ([len (string-length line)]) - (let loop ([start 7] [i 7] [best #f]) - (cond - [(> start len) best] - [(or (= i len) (char=? (string-ref line i) #\,)) - (let* ([item-end (python-import-trimmed-end line start i)] - [item (substring line start item-end)]) - (let-values ([(imported alias) - (split-alias - (python-import-clean-item item))]) - (loop - (+ i 1) - (+ i 1) - (if (and (not (string=? imported "")) - (python-import-item-matches-requirement? - imported - requirements)) - item-end - best))))] - [else (loop start (+ i 1) best)])))) - (def (python-import-line-match-end - source - first - line-end - requirements) - (let* ([line (substring source first line-end)] - [trimmed-line (string-trim line)]) - (if (sg-string-prefix? "import " trimmed-line) - (let ([relative-end (python-import-line-matching-import-end - trimmed-line - requirements)]) - (if relative-end - (+ first relative-end) - (python-import-trimmed-end source first line-end))) - (python-import-trimmed-end source first line-end)))) (def (insert-finding-by-start finding sorted) (cond [(null? sorted) (list finding)] @@ -22930,49 +22785,6 @@ (def (first-dotted-segment name) (let ([dot (first-index-of name #\.)]) (if dot (substring name 0 dot) name))) - (def (parse-python-import-items items imports) - (let loop ([xs (split-on-char items #\,)] [acc imports]) - (if (null? xs) - acc - (let-values ([(imported alias) (split-alias (car xs))]) - (loop - (cdr xs) - (add-name-import - (or alias (first-dotted-segment imported)) - imported - acc)))))) - (def (parse-python-from-import-items module items imports) - (let loop ([xs (split-on-char items #\,)] [acc imports]) - (if (null? xs) - acc - (let-values ([(imported alias) (split-alias (car xs))]) - (if (or (string=? imported "*") (string=? imported "")) - (loop (cdr xs) acc) - (loop - (cdr xs) - (add-name-import - (or alias imported) - (string-append module "." imported) - acc))))))) - (def (python-import-map source) - (let* ([from-imports (regex-fold-matches - "(^|\\n)\\s*from\\s+([A-Za-z_][A-Za-z0-9_\\.]*?)\\s+import\\s+([^\\n#]+)" - source - (lambda (match acc) - (parse-python-from-import-items - (re-match-group match 2) - (re-match-group match 3) - acc)) - '())] - [plain-imports (regex-fold-matches - "(^|\\n)\\s*import\\s+([^\\n#]+)" - source - (lambda (match acc) - (parse-python-import-items - (re-match-group match 2) - acc)) - from-imports)]) - plain-imports)) (def (parse-js-named-import-items module items imports) (let loop ([xs (split-on-char items #\,)] [acc imports]) (if (null? xs) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -45,7 +45,7 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "1330446113C8CD99") + ("src/semgrep/scan.ss" . "4E33249F5140DC38") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") ("src/semgrep/engine/py-constant-prop.ss" . @@ -57,7 +57,7 @@ ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/engine/py-import-scan.ss" . - "2CE1AD276EF87529") + "F1817B0AFD8D9500") ("src/semgrep/engine/ts-decorator-scan.ss" . "610AA3A90D6A95F8") --- a/src/semgrep/engine/py-import-scan.ss +++ b/src/semgrep/engine/py-import-scan.ss @@ -1,4 +1,6 @@ (export + python-import-map + python-import-clean-item trim-python-module-finding scan-python-import-local-pattern scan-python-import-local-pattern-with-bindings --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -12038,151 +12038,6 @@ (loop next (if finding (cons finding acc) acc)))))))) -(def (python-import-clean-item item) - (let ([without-delimiters - (let ([len (string-length item)]) - (let loop ([i 0] [acc '()]) - (cond - [(= i len) (list->string (reverse acc))] - [(let ([ch (string-ref item i)]) - (or (char=? ch #\() - (char=? ch #\)))) - (loop (+ i 1) acc)] - [else (loop (+ i 1) - (cons (string-ref item i) acc))])))]) - (string-trim without-delimiters))) - -(def (python-import-entry kind fqn) - (cons kind fqn)) - -(def (python-import-entry-kind entry) - (car entry)) - -(def (python-import-entry-fqn entry) - (cdr entry)) - -(def (python-import-items->entries items module) - (let loop ([xs (split-on-char items #\,)] [acc '()]) - (if (null? xs) - (reverse acc) - (let* ([clean (python-import-clean-item (car xs))]) - (let-values ([(imported alias) (split-alias clean)]) - (cond - [(string=? imported "") - (loop (cdr xs) acc)] - [(string=? imported "*") - (loop (cdr xs) - (if module - (cons (python-import-entry 'star module) acc) - acc))] - [else - (loop (cdr xs) - (cons (python-import-entry - 'normal - (if module - (string-append module "." imported) - imported)) - acc))])))))) - -(def (python-import-line-without-comment line) - (let ([comment (first-index-of line #\#)]) - (if comment - (substring line 0 comment) - line))) - -(def (python-absolute-module-name? module) - (and (> (string-length module) 0) - (not (char=? (string-ref module 0) #\.)))) - -(def (python-import-line-entries line) - (let* ([trimmed (string-trim (python-import-line-without-comment line))] - [len (string-length trimmed)]) - (cond - [(sg-string-prefix? "import " trimmed) - (python-import-items->entries - (substring trimmed 7 len) - #f)] - [(sg-string-prefix? "from " trimmed) - (let ([import-index (string-find-substring trimmed " import ")]) - (if import-index - (let ([module (string-trim (substring trimmed 5 import-index))]) - (if (python-absolute-module-name? module) - (python-import-items->entries - (substring trimmed (+ import-index 8) len) - module) - '())) - '()))] - [else '()]))) - -(def (python-import-pattern-requirements pattern) - (let ([trimmed (string-trim pattern)]) - (and (not (string-find-substring trimmed "\n")) - (not (string-find-substring trimmed "$")) - (not (string-find-substring trimmed "...")) - (let ([entries (python-import-line-entries trimmed)]) - (and (not (null? entries)) - (map python-import-entry-fqn entries)))))) - -(def (python-import-entry-satisfies? entry required) - (let ([fqn (python-import-entry-fqn entry)]) - (if (eq? (python-import-entry-kind entry) 'star) - (string=? fqn required) - (or (string=? fqn required) - (sg-string-prefix? (string-append required ".") fqn))))) - -(def (python-import-line-satisfies? entries requirements) - (all? (lambda (required) - (any? (lambda (entry) - (python-import-entry-satisfies? entry required)) - entries)) - requirements)) - -(def (python-import-trimmed-end source start end) - (let ([last (skip-horizontal-backward source (- end 1))]) - (cond - [(< last start) start] - [(char=? (string-ref source last) #\)) last] - [else (+ last 1)]))) - -(def (python-import-item-matches-requirement? imported requirements) - (let ([entry (python-import-entry 'normal imported)]) - (any? (lambda (required) - (python-import-entry-satisfies? entry required)) - requirements))) - -(def (python-import-line-matching-import-end line requirements) - (let ([len (string-length line)]) - (let loop ([start 7] [i 7] [best #f]) - (cond - [(> start len) best] - [(or (= i len) (char=? (string-ref line i) #\,)) - (let* ([item-end (python-import-trimmed-end line start i)] - [item (substring line start item-end)]) - (let-values ([(imported alias) - (split-alias (python-import-clean-item item))]) - (loop (+ i 1) - (+ i 1) - (if (and (not (string=? imported "")) - (python-import-item-matches-requirement? - imported - requirements)) - item-end - best))))] - [else (loop start (+ i 1) best)])))) - -(def (python-import-line-match-end source first line-end requirements) - (let* ([line (substring source first line-end)] - [trimmed-line (string-trim line)]) - (if (sg-string-prefix? "import " trimmed-line) - (let ([relative-end - (python-import-line-matching-import-end - trimmed-line - requirements)]) - (if relative-end - (+ first relative-end) - (python-import-trimmed-end source first line-end))) - (python-import-trimmed-end source first line-end)))) - (def (insert-finding-by-start finding sorted) (cond [(null? sorted) (list finding)] @@ -22830,50 +22685,6 @@ (let ([dot (first-index-of name #\.)]) (if dot (substring name 0 dot) name))) -(def (parse-python-import-items items imports) - (let loop ([xs (split-on-char items #\,)] [acc imports]) - (if (null? xs) - acc - (let-values ([(imported alias) (split-alias (car xs))]) - (loop (cdr xs) - (add-name-import (or alias (first-dotted-segment imported)) - imported - acc)))))) - -(def (parse-python-from-import-items module items imports) - (let loop ([xs (split-on-char items #\,)] [acc imports]) - (if (null? xs) - acc - (let-values ([(imported alias) (split-alias (car xs))]) - (if (or (string=? imported "*") - (string=? imported "")) - (loop (cdr xs) acc) - (loop (cdr xs) - (add-name-import - (or alias imported) - (string-append module "." imported) - acc))))))) - -(def (python-import-map source) - (let* ([from-imports - (regex-fold-matches - "(^|\\n)\\s*from\\s+([A-Za-z_][A-Za-z0-9_\\.]*?)\\s+import\\s+([^\\n#]+)" - source - (lambda (match acc) - (parse-python-from-import-items - (re-match-group match 2) - (re-match-group match 3) - acc)) - '())] - [plain-imports - (regex-fold-matches - "(^|\\n)\\s*import\\s+([^\\n#]+)" - source - (lambda (match acc) - (parse-python-import-items (re-match-group match 2) acc)) - from-imports)]) - plain-imports)) - (def (parse-js-named-import-items module items imports) (let loop ([xs (split-on-char items #\,)] [acc imports]) (if (null? xs)