Extract Semgrep TypeScript query scanners
ober
fd14c4ed99885bed83114168af67c8c05ec0877d
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -481,6 +481,8 @@ Completed in the repo: `src/semgrep/engine/ts-decorator-scan.ss` - extracted JavaScript decorator-method text scanning into `src/semgrep/engine/js-decorator-scan.ss` + - extracted TypeScript `req.params.id` and Mongo `$where` text scanning + into `src/semgrep/engine/ts-query-scan.ss` Validation at this checkpoint: new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/ts-query-scan.sls @@ -0,0 +1,155 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/ts-query-scan.ss + +(library (semgrep engine ts-query-scan) + (export + scan-typescript-req-param-assignment-pattern + scan-typescript-mongo-where-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (semgrep rule) (semgrep result) (semgrep result extras) + (semgrep engine regex-support) (semgrep source offsets) + (semgrep match structural)) + (def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + (def (typescript-req-param-assignment-pattern? pattern) + (and (string-find-substring pattern "$ASDF =") + (string-find-substring pattern "$REQ.params.id"))) + (define typescript-req-param-assignment-regex + "(?:const|let|var)?[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[^\\n]*([A-Za-z_$][A-Za-z0-9_$]*)\\.params\\.id") + (def (typescript-req-param-assignment-binding source) + (let ([match (re-search + (re typescript-req-param-assignment-regex) + source + 0)]) + (and match + (let* ([name (re-match-group match 1)] + [full (re-match-full match)] + [rel (or (string-find-substring-from full name 0) 0)] + [start (+ (re-match-start match) rel)] + [end (+ start (string-length name))]) + (make-regex-capture-binding "ASDF" name source start + end))))) + (def (scan-typescript-req-param-assignment-pattern + rule + path + source + pattern) + (and (typescript-req-param-assignment-pattern? pattern) + (let ([binding (typescript-req-param-assignment-binding + source)]) + (and binding + (let* ([bindings (list (cons "ASDF" binding))] + [message (render-fix-template + (rule-message rule) + bindings)] + [extra (finding-extra-for-match + rule + bindings + source)] + [end (string-length source)]) + (let-values ([(end-line end-col) + (offset->line-col source end)]) + (list + (make-finding (rule-id rule) path 1 1 end-line + end-col 0 end message (rule-severity rule) + extra)))))))) + (def (typescript-mongo-where-pattern? pattern) + (and (string-find-substring pattern "$DB.orders.find") + (string-find-substring pattern "$where") + (string-find-substring pattern "$ASDF"))) + (def (identifier-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$))) + (def (identifier-token-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_))) + (def (token-start-boundary? source index) + (or (= index 0) + (not (identifier-token-char? + (string-ref source (- index 1)))))) + (def (token-end-boundary? source index) + (or (= index (string-length source)) + (not (identifier-token-char? (string-ref source index))))) + (def (string-contains-token? source token) + (let ([token-len (string-length token)]) + (and (> token-len 0) + (let loop ([start 0]) + (let ([index (string-find-substring-from + source + token + start)]) + (and index + (let ([end (+ index token-len)]) + (if (and (token-start-boundary? source index) + (token-end-boundary? source end)) + #t + (loop (+ index 1)))))))))) + (def (identifier-start-before source index) + (let loop ([i (- index 1)]) + (if (and (>= i 0) (identifier-char? (string-ref source i))) + (loop (- i 1)) + (+ i 1)))) + (def (scan-typescript-mongo-where-pattern + rule + path + source + pattern) + (and (typescript-mongo-where-pattern? pattern) + (let* ([binding (typescript-req-param-assignment-binding + source)] + [name (and binding (metavariable-binding-text binding))] + [member-index (string-find-substring + source + ".orders.find(")] + [start (and member-index + (identifier-start-before + source + member-index))] + [then-index (and start + (string-find-substring-from + source + ".then" + start))] + [end then-index]) + (and binding + start + end + (let ([call-text (substring source start end)]) + (and (string-find-substring call-text "$where") + (or (string-find-substring + call-text + (string-append "${" name "}")) + (string-contains-token? call-text name)) + (let* ([bindings (list (cons "ASDF" binding))] + [message (render-fix-template + (rule-message rule) + bindings)] + [extra (finding-extra-for-match + rule + bindings + call-text)]) + (let-values ([(start-line start-col) + (offset->line-col source start)] + [(end-line end-col) + (offset->line-col source end)]) + (list + (make-finding (rule-id rule) path start-line start-col + end-line end-col start end message + (rule-severity rule) extra))))))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -23,10 +23,10 @@ (semgrep engine markup-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) - (semgrep engine text-support) (semgrep rule parse-rule) - (semgrep parse parse-target) (semgrep source offsets) - (semgrep targeting path-filter) (semgrep util literals) - (semgrep match structural)) + (semgrep engine ts-query-scan) (semgrep engine text-support) + (semgrep rule parse-rule) (semgrep parse parse-target) + (semgrep source offsets) (semgrep targeting path-filter) + (semgrep util literals) (semgrep match structural)) (def (alist-ref/default xs key default) (let ([found (assoc key xs)]) (if found (cdr found) default))) @@ -616,104 +616,6 @@ extra) acc))))))))))) (decorated-class-ranges source)))))) - (def (typescript-req-param-assignment-pattern? pattern) - (and (string-find-substring pattern "$ASDF =") - (string-find-substring pattern "$REQ.params.id"))) - (def typescript-req-param-assignment-regex - "(?:const|let|var)?[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[^\\n]*([A-Za-z_$][A-Za-z0-9_$]*)\\.params\\.id") - (def (typescript-req-param-assignment-binding source) - (let ([match (re-search - (re typescript-req-param-assignment-regex) - source - 0)]) - (and match - (let* ([name (re-match-group match 1)] - [full (re-match-full match)] - [rel (or (string-find-substring-from full name 0) 0)] - [start (+ (re-match-start match) rel)] - [end (+ start (string-length name))]) - (make-regex-capture-binding "ASDF" name source start - end))))) - (def (scan-typescript-req-param-assignment-pattern - rule - path - source - pattern) - (and (typescript-req-param-assignment-pattern? pattern) - (let ([binding (typescript-req-param-assignment-binding - source)]) - (and binding - (let* ([bindings (list (cons "ASDF" binding))] - [message (render-fix-template - (rule-message rule) - bindings)] - [extra (finding-extra-for-match - rule - bindings - source)] - [end (string-length source)]) - (let-values ([(end-line end-col) - (offset->line-col source end)]) - (list - (make-finding (rule-id rule) path 1 1 end-line - end-col 0 end message (rule-severity rule) - extra)))))))) - (def (typescript-mongo-where-pattern? pattern) - (and (string-find-substring pattern "$DB.orders.find") - (string-find-substring pattern "$where") - (string-find-substring pattern "$ASDF"))) - (def (identifier-start-before source index) - (let loop ([i (- index 1)]) - (if (and (>= i 0) (identifier-char? (string-ref source i))) - (loop (- i 1)) - (+ i 1)))) - (def (scan-typescript-mongo-where-pattern - rule - path - source - pattern) - (and (typescript-mongo-where-pattern? pattern) - (let* ([binding (typescript-req-param-assignment-binding - source)] - [name (and binding (metavariable-binding-text binding))] - [member-index (string-find-substring - source - ".orders.find(")] - [start (and member-index - (identifier-start-before - source - member-index))] - [then-index (and start - (string-find-substring-from - source - ".then" - start))] - [end then-index]) - (and binding - start - end - (let ([call-text (substring source start end)]) - (and (string-find-substring call-text "$where") - (or (string-find-substring - call-text - (string-append "${" name "}")) - (string-contains-token? call-text name)) - (let* ([bindings (list (cons "ASDF" binding))] - [message (render-fix-template - (rule-message rule) - bindings)] - [extra (finding-extra-for-match - rule - bindings - call-text)]) - (let-values ([(start-line start-col) - (offset->line-col source start)] - [(end-line end-col) - (offset->line-col source end)]) - (list - (make-finding (rule-id rule) path start-line start-col - end-line end-col start end message - (rule-severity rule) extra)))))))))) (def (javascript-new-function-sequence-pattern? pattern) (and (string-find-substring pattern "new Function(") (string-find-substring pattern "$FUNC();"))) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -1,35 +1,36 @@ -(("src/semgrep/engine/js-vardef-scan.ss" +(("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") + ("src/semgrep/engine/js-vardef-scan.ss" . "BD82BDDFEDD7242D") - ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") - ("src/semgrep/engine/regex-scan.ss" . "D75414F0AEFE2F66") ("src/semgrep/result.ss" . "22D23E40B49BA529") + ("src/semgrep/engine/regex-scan.ss" . "D75414F0AEFE2F66") ("src/semgrep/engine/rule-plan.ss" . "6631789392AB5F80") ("src/semgrep/result/findings.ss" . "547811661239D9C7") ("src/semgrep/util/literals.ss" . "8A094085551B216E") - ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/engine/ts-query-scan.ss" . "51AD5339F6DE47B4") ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") ("src/semgrep/rule/parse-rule.ss" . "EFA6D401699CEDEF") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") ("src/semgrep/result/extras.ss" . "DF0B3AAE2BAEB5D") + ("src/semgrep/scan.ss" . "94E771EEFE98041B") ("src/semgrep/engine/js-decorator-scan.ss" . "193758AD2E444FD6") - ("src/semgrep/scan.ss" . "848B40CD850A0B6D") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/rule.ss" . "E12C108153C181FA") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/rule.ss" . "E12C108153C181FA") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") + ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") + ("src/semgrep/engine/text-support.ss" . "644AF29394C53045") ("src/semgrep/engine/ts-decorator-scan.ss" . "610AA3A90D6A95F8") - ("src/semgrep/engine/text-support.ss" . "644AF29394C53045") - ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/ts-query-scan.ss @@ -0,0 +1,159 @@ +(export + scan-typescript-req-param-assignment-pattern + scan-typescript-mongo-where-pattern) + +(import (except (jerboa prelude) meta atom?) + (std regex) + (semgrep rule) + (semgrep result) + (semgrep result extras) + (semgrep engine regex-support) + (semgrep source offsets) + (semgrep match structural)) + +(def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + +(def (typescript-req-param-assignment-pattern? pattern) + (and (string-find-substring pattern "$ASDF =") + (string-find-substring pattern "$REQ.params.id"))) + +(define typescript-req-param-assignment-regex + "(?:const|let|var)?[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[^\\n]*([A-Za-z_$][A-Za-z0-9_$]*)\\.params\\.id") + +(def (typescript-req-param-assignment-binding source) + (let ([match (re-search (re typescript-req-param-assignment-regex) + source + 0)]) + (and match + (let* ([name (re-match-group match 1)] + [full (re-match-full match)] + [rel (or (string-find-substring-from full name 0) 0)] + [start (+ (re-match-start match) rel)] + [end (+ start (string-length name))]) + (make-regex-capture-binding "ASDF" name source start end))))) + +(def (scan-typescript-req-param-assignment-pattern rule path source pattern) + (and (typescript-req-param-assignment-pattern? pattern) + (let ([binding (typescript-req-param-assignment-binding source)]) + (and binding + (let* ([bindings (list (cons "ASDF" binding))] + [message (render-fix-template + (rule-message rule) + bindings)] + [extra (finding-extra-for-match rule bindings source)] + [end (string-length source)]) + (let-values ([(end-line end-col) + (offset->line-col source end)]) + (list + (make-finding + (rule-id rule) + path + 1 + 1 + end-line + end-col + 0 + end + message + (rule-severity rule) + extra)))))))) + +(def (typescript-mongo-where-pattern? pattern) + (and (string-find-substring pattern "$DB.orders.find") + (string-find-substring pattern "$where") + (string-find-substring pattern "$ASDF"))) + +(def (identifier-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_) + (char=? ch #\$))) + +(def (identifier-token-char? ch) + (or (char-alphabetic? ch) + (char-numeric? ch) + (char=? ch #\_))) + +(def (token-start-boundary? source index) + (or (= index 0) + (not (identifier-token-char? (string-ref source (- index 1)))))) + +(def (token-end-boundary? source index) + (or (= index (string-length source)) + (not (identifier-token-char? (string-ref source index))))) + +(def (string-contains-token? source token) + (let ([token-len (string-length token)]) + (and (> token-len 0) + (let loop ([start 0]) + (let ([index (string-find-substring-from source token start)]) + (and index + (let ([end (+ index token-len)]) + (if (and (token-start-boundary? source index) + (token-end-boundary? source end)) + #t + (loop (+ index 1)))))))))) + +(def (identifier-start-before source index) + (let loop ([i (- index 1)]) + (if (and (>= i 0) + (identifier-char? (string-ref source i))) + (loop (- i 1)) + (+ i 1)))) + +(def (scan-typescript-mongo-where-pattern rule path source pattern) + (and (typescript-mongo-where-pattern? pattern) + (let* ([binding (typescript-req-param-assignment-binding source)] + [name (and binding (metavariable-binding-text binding))] + [member-index (string-find-substring source ".orders.find(")] + [start (and member-index + (identifier-start-before source member-index))] + [then-index (and start + (string-find-substring-from + source + ".then" + start))] + [end then-index]) + (and binding + start + end + (let ([call-text (substring source start end)]) + (and (string-find-substring call-text "$where") + (or (string-find-substring + call-text + (string-append "${" name "}")) + (string-contains-token? call-text name)) + (let* ([bindings (list (cons "ASDF" binding))] + [message (render-fix-template + (rule-message rule) + bindings)] + [extra (finding-extra-for-match + rule + bindings + call-text)]) + (let-values ([(start-line start-col) + (offset->line-col source start)] + [(end-line end-col) + (offset->line-col source end)]) + (list + (make-finding + (rule-id rule) + path + start-line + start-col + end-line + end-col + start + end + message + (rule-severity rule) + extra)))))))))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -21,6 +21,7 @@ (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) + (semgrep engine ts-query-scan) (semgrep engine text-support) (semgrep rule parse-rule) (semgrep parse parse-target) @@ -657,111 +658,6 @@ acc))))))))))) (decorated-class-ranges source)))))) -(def (typescript-req-param-assignment-pattern? pattern) - (and (string-find-substring pattern "$ASDF =") - (string-find-substring pattern "$REQ.params.id"))) - -(def typescript-req-param-assignment-regex - "(?:const|let|var)?[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[^\\n]*([A-Za-z_$][A-Za-z0-9_$]*)\\.params\\.id") - -(def (typescript-req-param-assignment-binding source) - (let ([match (re-search (re typescript-req-param-assignment-regex) - source - 0)]) - (and match - (let* ([name (re-match-group match 1)] - [full (re-match-full match)] - [rel (or (string-find-substring-from full name 0) 0)] - [start (+ (re-match-start match) rel)] - [end (+ start (string-length name))]) - (make-regex-capture-binding "ASDF" name source start end))))) - -(def (scan-typescript-req-param-assignment-pattern rule path source pattern) - (and (typescript-req-param-assignment-pattern? pattern) - (let ([binding (typescript-req-param-assignment-binding source)]) - (and binding - (let* ([bindings (list (cons "ASDF" binding))] - [message (render-fix-template - (rule-message rule) - bindings)] - [extra (finding-extra-for-match rule bindings source)] - [end (string-length source)]) - (let-values ([(end-line end-col) - (offset->line-col source end)]) - (list - (make-finding - (rule-id rule) - path - 1 - 1 - end-line - end-col - 0 - end - message - (rule-severity rule) - extra)))))))) - -(def (typescript-mongo-where-pattern? pattern) - (and (string-find-substring pattern "$DB.orders.find") - (string-find-substring pattern "$where") - (string-find-substring pattern "$ASDF"))) - -(def (identifier-start-before source index) - (let loop ([i (- index 1)]) - (if (and (>= i 0) - (identifier-char? (string-ref source i))) - (loop (- i 1)) - (+ i 1)))) - -(def (scan-typescript-mongo-where-pattern rule path source pattern) - (and (typescript-mongo-where-pattern? pattern) - (let* ([binding (typescript-req-param-assignment-binding source)] - [name (and binding (metavariable-binding-text binding))] - [member-index (string-find-substring source ".orders.find(")] - [start (and member-index - (identifier-start-before source member-index))] - [then-index (and start - (string-find-substring-from - source - ".then" - start))] - [end then-index]) - (and binding - start - end - (let ([call-text (substring source start end)]) - (and (string-find-substring call-text "$where") - (or (string-find-substring - call-text - (string-append "${" name "}")) - (string-contains-token? call-text name)) - (let* ([bindings (list (cons "ASDF" binding))] - [message (render-fix-template - (rule-message rule) - bindings)] - [extra (finding-extra-for-match - rule - bindings - call-text)]) - (let-values ([(start-line start-col) - (offset->line-col source start)] - [(end-line end-col) - (offset->line-col source end)]) - (list - (make-finding - (rule-id rule) - path - start-line - start-col - end-line - end-col - start - end - message - (rule-severity rule) - extra)))))))))) - (def (javascript-new-function-sequence-pattern? pattern) (and (string-find-substring pattern "new Function(") (string-find-substring pattern "$FUNC();")))