Extract Semgrep Python string evaluators
ober
71a93302139e1db7eb7a5e9c987a00c366254065
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -494,6 +494,8 @@ Completed in the repo: `src/semgrep/engine/py-constant-scan.ss` - extracted Python string-call pattern parsing and scan dispatch into `src/semgrep/engine/py-string-scan.ss` + - extracted reusable Python string/f-string literal evaluation helpers into + `src/semgrep/engine/py-string-eval.ss` Validation at this checkpoint: new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/py-string-eval.sls @@ -0,0 +1,69 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/py-string-eval.ss + +(library (semgrep engine py-string-eval) + (export + python-fstring-literal? + python-string-literal? + python-constant-string-literal-value*) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) + (semgrep util literals) + (semgrep engine text-support)) + (def (python-fstring-literal? text) + (let* ([trimmed (string-trim text)] + [len (string-length trimmed)]) + (and (>= len 3) + (let ([prefix (string-ref trimmed 0)] + [quote (string-ref trimmed 1)]) + (and (or (char=? prefix #\f) (char=? prefix #\F)) + (or (char=? quote #\") (char=? quote #\')) + (char=? (string-ref trimmed (- len 1)) quote)))))) + (def (python-string-literal? text) + (let ([trimmed (string-trim text)]) + (or (quoted-string? trimmed) + (python-fstring-literal? trimmed)))) + (def (python-fstring-content-value* + content + source + before-offset + resolve-expression) + (let ([len (string-length content)]) + (let loop ([i 0] [acc '()]) + (cond + [(= i len) (list->string (reverse acc))] + [(char=? (string-ref content i) #\{) + (let ([end (char-index-from content #\} (+ i 1) len)]) + (and end + (let ([value (resolve-expression + (substring content (+ i 1) end) + source + before-offset)]) + (and value + (loop + (+ end 1) + (append + (reverse (string->list value)) + acc))))))] + [else (loop (+ i 1) (cons (string-ref content i) acc))])))) + (def (python-constant-string-literal-value* + text + source + before-offset + resolve-expression) + (let* ([trimmed (string-trim text)] + [len (string-length trimmed)]) + (cond + [(quoted-string? trimmed) (unquote-string trimmed)] + [(python-fstring-literal? trimmed) + (python-fstring-content-value* + (substring trimmed 2 (- len 1)) + source + before-offset + resolve-expression)] + [else #f])))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -24,6 +24,7 @@ (semgrep engine js-decorator-scan) (semgrep engine markup-scan) (semgrep engine py-constant-scan) + (semgrep engine py-string-eval) (semgrep engine py-string-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) @@ -530,19 +531,6 @@ (char=? ch #\newline)))) (loop (+ j 1)) j))) - (def (python-fstring-literal? text) - (let* ([trimmed (string-trim text)] - [len (string-length trimmed)]) - (and (>= len 3) - (let ([prefix (string-ref trimmed 0)] - [quote (string-ref trimmed 1)]) - (and (or (char=? prefix #\f) (char=? prefix #\F)) - (or (char=? quote #\") (char=? quote #\')) - (char=? (string-ref trimmed (- len 1)) quote)))))) - (def (python-string-literal? text) - (let ([trimmed (string-trim text)]) - (or (quoted-string? trimmed) - (python-fstring-literal? trimmed)))) (def (python-fstring-expression-string expr source @@ -563,42 +551,15 @@ [value (comparison-value trimmed constants #f #f)]) (and (not (comparison-missing? value)) (comparison-value->string value)))]))) - (def (python-fstring-content-value - content - source - before-offset) - (let ([len (string-length content)]) - (let loop ([i 0] [acc '()]) - (cond - [(= i len) (list->string (reverse acc))] - [(char=? (string-ref content i) #\{) - (let ([end (char-index-from content #\} (+ i 1) len)]) - (and end - (let ([value (python-fstring-expression-string - (substring content (+ i 1) end) - source - before-offset)]) - (and value - (loop - (+ end 1) - (append - (reverse (string->list value)) - acc))))))] - [else (loop (+ i 1) (cons (string-ref content i) acc))])))) (def (python-constant-string-literal-value text source before-offset) - (let* ([trimmed (string-trim text)] - [len (string-length trimmed)]) - (cond - [(quoted-string? trimmed) (unquote-string trimmed)] - [(python-fstring-literal? trimmed) - (python-fstring-content-value - (substring trimmed 2 (- len 1)) - source - before-offset)] - [else #f]))) + (python-constant-string-literal-value* + text + source + before-offset + python-fstring-expression-string)) (def (scan-python-fstring-call-pattern rule path --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -5,6 +5,9 @@ ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/engine/py-string-eval.ss" + . + "390E770972B5028B") ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") ("src/semgrep/engine/js-eval-scan.ss" . "916D40FD4680B9BD") ("src/semgrep/result/extras.ss" . "DF0B3AAE2BAEB5D") @@ -17,10 +20,10 @@ ("src/semgrep/engine/ts-type-scan.ss" . "B36BAF07D19F4419") ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") - ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") ("src/semgrep/engine/py-constant-scan.ss" . "285FDB423AD7E28C") + ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC") ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") ("src/semgrep/engine/js-vardef-scan.ss" @@ -35,7 +38,7 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "912D7E4F9F9BA531") + ("src/semgrep/scan.ss" . "B9A67A34730D53BE") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") ("src/semgrep/engine/js-constructor-scan.ss" . new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/py-string-eval.ss @@ -0,0 +1,58 @@ +(export + python-fstring-literal? + python-string-literal? + python-constant-string-literal-value*) + +(import (except (jerboa prelude) meta atom?) + (semgrep util literals) + (semgrep engine text-support)) + +(def (python-fstring-literal? text) + (let* ([trimmed (string-trim text)] + [len (string-length trimmed)]) + (and (>= len 3) + (let ([prefix (string-ref trimmed 0)] + [quote (string-ref trimmed 1)]) + (and (or (char=? prefix #\f) (char=? prefix #\F)) + (or (char=? quote #\") (char=? quote #\')) + (char=? (string-ref trimmed (- len 1)) quote)))))) + +(def (python-string-literal? text) + (let ([trimmed (string-trim text)]) + (or (quoted-string? trimmed) + (python-fstring-literal? trimmed)))) + +(def (python-fstring-content-value* content source before-offset resolve-expression) + (let ([len (string-length content)]) + (let loop ([i 0] [acc '()]) + (cond + [(= i len) (list->string (reverse acc))] + [(char=? (string-ref content i) #\{) + (let ([end (char-index-from content #\} (+ i 1) len)]) + (and end + (let ([value (resolve-expression + (substring content (+ i 1) end) + source + before-offset)]) + (and value + (loop (+ end 1) + (append (reverse (string->list value)) + acc))))))] + [else (loop (+ i 1) (cons (string-ref content i) acc))])))) + +(def (python-constant-string-literal-value* + text + source + before-offset + resolve-expression) + (let* ([trimmed (string-trim text)] + [len (string-length trimmed)]) + (cond + [(quoted-string? trimmed) (unquote-string trimmed)] + [(python-fstring-literal? trimmed) + (python-fstring-content-value* + (substring trimmed 2 (- len 1)) + source + before-offset + resolve-expression)] + [else #f]))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -20,6 +20,7 @@ (semgrep engine js-decorator-scan) (semgrep engine markup-scan) (semgrep engine py-constant-scan) + (semgrep engine py-string-eval) (semgrep engine py-string-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) @@ -584,21 +585,6 @@ (loop (+ j 1)) j))) -(def (python-fstring-literal? text) - (let* ([trimmed (string-trim text)] - [len (string-length trimmed)]) - (and (>= len 3) - (let ([prefix (string-ref trimmed 0)] - [quote (string-ref trimmed 1)]) - (and (or (char=? prefix #\f) (char=? prefix #\F)) - (or (char=? quote #\") (char=? quote #\')) - (char=? (string-ref trimmed (- len 1)) quote)))))) - -(def (python-string-literal? text) - (let ([trimmed (string-trim text)]) - (or (quoted-string? trimmed) - (python-fstring-literal? trimmed)))) - (def (python-fstring-expression-string expr source before-offset) (let ([trimmed (string-trim expr)]) (cond @@ -611,35 +597,12 @@ (and (not (comparison-missing? value)) (comparison-value->string value)))]))) -(def (python-fstring-content-value content source before-offset) - (let ([len (string-length content)]) - (let loop ([i 0] [acc '()]) - (cond - [(= i len) (list->string (reverse acc))] - [(char=? (string-ref content i) #\{) - (let ([end (char-index-from content #\} (+ i 1) len)]) - (and end - (let ([value (python-fstring-expression-string - (substring content (+ i 1) end) - source - before-offset)]) - (and value - (loop (+ end 1) - (append (reverse (string->list value)) - acc))))))] - [else (loop (+ i 1) (cons (string-ref content i) acc))])))) - (def (python-constant-string-literal-value text source before-offset) - (let* ([trimmed (string-trim text)] - [len (string-length trimmed)]) - (cond - [(quoted-string? trimmed) (unquote-string trimmed)] - [(python-fstring-literal? trimmed) - (python-fstring-content-value - (substring trimmed 2 (- len 1)) - source - before-offset)] - [else #f]))) + (python-constant-string-literal-value* + text + source + before-offset + python-fstring-expression-string)) (def (scan-python-fstring-call-pattern rule path source pattern) (scan-python-string-call-pattern