Extract Semgrep TypeScript type scanners
ober
34c3a81b7d1e55b4abce56a6e4aaa7afebd99ee3
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -487,6 +487,8 @@ Completed in the repo: into `src/semgrep/engine/js-constructor-scan.ss` - extracted JavaScript `new Function(...)` sequence/static text scanning into `src/semgrep/engine/js-eval-scan.ss` + - extracted TypeScript `import = require(...)` and object-type text + scanning into `src/semgrep/engine/ts-type-scan.ss` Validation at this checkpoint: new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/ts-type-scan.sls @@ -0,0 +1,136 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/ts-type-scan.ss + +(library (semgrep engine ts-type-scan) + (export + scan-typescript-import-equals-pattern + scan-typescript-object-type-pattern) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (std regex) + (semgrep rule) (semgrep result) (semgrep result extras) + (semgrep engine regex-scan) (semgrep engine regex-support) + (semgrep source offsets) (semgrep match structural)) + (def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + (def (typescript-import-equals-pattern? pattern) + (and (string-find-substring pattern "import $I =") + (string-find-substring pattern "$FOO"))) + (def (scan-typescript-import-equals-pattern + rule + path + source + pattern) + (and (typescript-import-equals-pattern? pattern) + (let ([rx (re "\\bimport[ \\t]+[A-Za-z_$][A-Za-z0-9_$]*[ \\t]*=[ \\t]*require[ \\t]*\\([^\\n)]*")] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop + next + (cons + (finding-from-match rule path source match) + acc))) + (reverse acc)))))))) + (def (typescript-object-type-pattern? pattern) + (and (string-find-substring pattern "type $TYPENAME") + (string-find-substring pattern "$TYPE &") + (string-find-substring pattern "$REGEX: string"))) + (define typescript-object-type-regex + "\\btype[ \\t]+([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[ \\t]*([^&;]+)[ \\t]*&[ \\t]*\\{[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*:[ \\t]*string[ \\t]*\\}[ \\t]*;?") + (def (typescript-object-type-finding rule path source match) + (let* ([typename (re-match-group match 1)] + [type-text (string-trim (re-match-group match 2))] + [regex-name (re-match-group match 3)] + [full (re-match-full match)] + [start (re-match-start match)] + [end (re-match-end match)] + [typename-rel (or (string-find-substring-from + full + typename + 0) + 0)] + [type-rel (or (string-find-substring-from + full + type-text + (+ typename-rel (string-length typename))) + typename-rel)] + [regex-rel (or (string-find-substring-from + full + regex-name + (+ type-rel (string-length type-text))) + type-rel)] + [bindings (list + (cons + "TYPENAME" + (make-regex-capture-binding "TYPENAME" typename source + (+ start typename-rel) + (+ start + typename-rel + (string-length typename)))) + (cons + "TYPE" + (make-regex-capture-binding "TYPE" type-text source (+ start type-rel) + (+ start + type-rel + (string-length type-text)))) + (cons + "REGEX" + (make-regex-capture-binding "REGEX" regex-name source (+ start regex-rel) + (+ start + regex-rel + (string-length regex-name)))))] + [extra (finding-extra-for-match + rule + bindings + (substring source start end))] + [message (render-fix-template + (rule-message rule) + bindings)]) + (let-values ([(start-line start-col) + (offset->line-col source start)] + [(end-line end-col) (offset->line-col source end)]) + (make-finding (rule-id rule) path start-line start-col + end-line end-col start end message (rule-severity rule) + extra)))) + (def (scan-typescript-object-type-pattern + rule + path + source + pattern) + (and (typescript-object-type-pattern? pattern) + (let ([rx (re typescript-object-type-regex)] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search rx source start)]) + (if match + (loop + (max (+ (re-match-start match) 1) + (re-match-end match)) + (cons + (typescript-object-type-finding + rule + path + source + match) + acc)) + (reverse acc))))))))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -25,10 +25,11 @@ (semgrep engine markup-scan) (semgrep engine regex-scan) (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) - (semgrep engine ts-query-scan) (semgrep engine text-support) - (semgrep rule parse-rule) (semgrep parse parse-target) - (semgrep source offsets) (semgrep targeting path-filter) - (semgrep util literals) (semgrep match structural)) + (semgrep engine ts-type-scan) (semgrep engine ts-query-scan) + (semgrep engine text-support) (semgrep rule parse-rule) + (semgrep parse parse-target) (semgrep source offsets) + (semgrep targeting path-filter) (semgrep util literals) + (semgrep match structural)) (def (alist-ref/default xs key default) (let ([found (assoc key xs)]) (if found (cdr found) default))) @@ -16955,115 +16956,6 @@ (loop (cdr candidates) (if finding (cons finding acc) acc))))))) - (def (typescript-import-equals-pattern? pattern) - (and (string-find-substring pattern "import $I =") - (string-find-substring pattern "$FOO"))) - (def (scan-typescript-import-equals-pattern - rule - path - source - pattern) - (and (typescript-import-equals-pattern? pattern) - (let ([rx (re "\\bimport[ \\t]+[A-Za-z_$][A-Za-z0-9_$]*[ \\t]*=[ \\t]*require[ \\t]*\\([^\\n)]*")] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search rx source start)]) - (if match - (let ([next (max (+ (re-match-start match) 1) - (re-match-end match))]) - (loop - next - (cons - (finding-from-match rule path source match) - acc))) - (reverse acc)))))))) - (def (typescript-object-type-pattern? pattern) - (and (string-find-substring pattern "type $TYPENAME") - (string-find-substring pattern "$TYPE &") - (string-find-substring pattern "$REGEX: string"))) - (def typescript-object-type-regex - "\\btype[ \\t]+([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[ \\t]*([^&;]+)[ \\t]*&[ \\t]*\\{[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*:[ \\t]*string[ \\t]*\\}[ \\t]*;?") - (def (typescript-object-type-finding rule path source match) - (let* ([typename (re-match-group match 1)] - [type-text (string-trim (re-match-group match 2))] - [regex-name (re-match-group match 3)] - [full (re-match-full match)] - [start (re-match-start match)] - [end (re-match-end match)] - [typename-rel (or (string-find-substring-from - full - typename - 0) - 0)] - [type-rel (or (string-find-substring-from - full - type-text - (+ typename-rel (string-length typename))) - typename-rel)] - [regex-rel (or (string-find-substring-from - full - regex-name - (+ type-rel (string-length type-text))) - type-rel)] - [bindings (list - (cons - "TYPENAME" - (make-regex-capture-binding "TYPENAME" typename source - (+ start typename-rel) - (+ start - typename-rel - (string-length typename)))) - (cons - "TYPE" - (make-regex-capture-binding "TYPE" type-text source (+ start type-rel) - (+ start - type-rel - (string-length type-text)))) - (cons - "REGEX" - (make-regex-capture-binding "REGEX" regex-name source (+ start regex-rel) - (+ start - regex-rel - (string-length regex-name)))))] - [extra (finding-extra-for-match - rule - bindings - (substring source start end))] - [message (render-fix-template - (rule-message rule) - bindings)]) - (let-values ([(start-line start-col) - (offset->line-col source start)] - [(end-line end-col) (offset->line-col source end)]) - (make-finding (rule-id rule) path start-line start-col - end-line end-col start end message (rule-severity rule) - extra)))) - (def (scan-typescript-object-type-pattern - rule - path - source - pattern) - (and (typescript-object-type-pattern? pattern) - (let ([rx (re typescript-object-type-regex)] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search rx source start)]) - (if match - (loop - (max (+ (re-match-start match) 1) - (re-match-end match)) - (cons - (typescript-object-type-finding - rule - path - source - match) - acc)) - (reverse acc)))))))) (def (javascript-imported-member-call-pattern? pattern) (let ([trimmed (string-trim pattern)]) (and (string-find-substring --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -1,4 +1,22 @@ -(("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") +(("src/semgrep/util/literals.ss" . "8A094085551B216E") + ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") + ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") + ("src/semgrep/engine/js-eval-scan.ss" . "916D40FD4680B9BD") + ("src/semgrep/result/extras.ss" . "DF0B3AAE2BAEB5D") + ("src/semgrep/engine/js-decorator-scan.ss" + . + "193758AD2E444FD6") + ("src/semgrep/engine/ts-type-scan.ss" . "B36BAF07D19F4419") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/rule.ss" . "E12C108153C181FA") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/source/offsets.ss" . "834EFDB706823794") + ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") + ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") + ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC") + ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") ("src/semgrep/engine/js-vardef-scan.ss" . "BD82BDDFEDD7242D") @@ -6,35 +24,18 @@ ("src/semgrep/engine/regex-scan.ss" . "D75414F0AEFE2F66") ("src/semgrep/engine/rule-plan.ss" . "6631789392AB5F80") ("src/semgrep/result/findings.ss" . "547811661239D9C7") - ("src/semgrep/util/literals.ss" . "8A094085551B216E") - ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") - ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") ("src/semgrep/engine/ts-query-scan.ss" . "51AD5339F6DE47B4") - ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") - ("src/semgrep/engine/js-eval-scan.ss" . "916D40FD4680B9BD") ("src/semgrep/rule/parse-rule.ss" . "EFA6D401699CEDEF") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/result/extras.ss" . "DF0B3AAE2BAEB5D") - ("src/semgrep/scan.ss" . "88CD1FC263753713") - ("src/semgrep/engine/js-decorator-scan.ss" - . - "193758AD2E444FD6") + ("src/semgrep/scan.ss" . "676548090DED3083") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") - ("src/semgrep/rule.ss" . "E12C108153C181FA") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/engine/js-constructor-scan.ss" . "6B226B8D0584A7A") - ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") - ("src/semgrep/result/builders.ss" . "93A64AF4435E6132") ("src/semgrep/engine/text-support.ss" . "644AF29394C53045") ("src/semgrep/engine/ts-decorator-scan.ss" . - "610AA3A90D6A95F8") - ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) + "610AA3A90D6A95F8")) new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/ts-type-scan.ss @@ -0,0 +1,136 @@ +(export + scan-typescript-import-equals-pattern + scan-typescript-object-type-pattern) + +(import (except (jerboa prelude) meta atom?) + (std regex) + (semgrep rule) + (semgrep result) + (semgrep result extras) + (semgrep engine regex-scan) + (semgrep engine regex-support) + (semgrep source offsets) + (semgrep match structural)) + +(def (string-find-substring s needle) + (let ([len (string-length s)] + [needle-len (string-length needle)]) + (let loop ([i 0]) + (cond + [(> i len) #f] + [(and (<= (+ i needle-len) len) + (string=? (substring s i (+ i needle-len)) needle)) + i] + [else (loop (+ i 1))])))) + +(def (typescript-import-equals-pattern? pattern) + (and (string-find-substring pattern "import $I =") + (string-find-substring pattern "$FOO"))) + +(def (scan-typescript-import-equals-pattern rule path source pattern) + (and (typescript-import-equals-pattern? pattern) + (let ([rx (re "\\bimport[ \\t]+[A-Za-z_$][A-Za-z0-9_$]*[ \\t]*=[ \\t]*require[ \\t]*\\([^\\n)]*")] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search rx source start)]) + (if match + (let ([next (max (+ (re-match-start match) 1) + (re-match-end match))]) + (loop next + (cons (finding-from-match + rule + path + source + match) + acc))) + (reverse acc)))))))) + +(def (typescript-object-type-pattern? pattern) + (and (string-find-substring pattern "type $TYPENAME") + (string-find-substring pattern "$TYPE &") + (string-find-substring pattern "$REGEX: string"))) + +(define typescript-object-type-regex + "\\btype[ \\t]+([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[ \\t]*([^&;]+)[ \\t]*&[ \\t]*\\{[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*:[ \\t]*string[ \\t]*\\}[ \\t]*;?") + +(def (typescript-object-type-finding rule path source match) + (let* ([typename (re-match-group match 1)] + [type-text (string-trim (re-match-group match 2))] + [regex-name (re-match-group match 3)] + [full (re-match-full match)] + [start (re-match-start match)] + [end (re-match-end match)] + [typename-rel (or (string-find-substring-from full typename 0) 0)] + [type-rel (or (string-find-substring-from + full + type-text + (+ typename-rel (string-length typename))) + typename-rel)] + [regex-rel (or (string-find-substring-from + full + regex-name + (+ type-rel (string-length type-text))) + type-rel)] + [bindings + (list + (cons "TYPENAME" + (make-regex-capture-binding + "TYPENAME" + typename + source + (+ start typename-rel) + (+ start typename-rel (string-length typename)))) + (cons "TYPE" + (make-regex-capture-binding + "TYPE" + type-text + source + (+ start type-rel) + (+ start type-rel (string-length type-text)))) + (cons "REGEX" + (make-regex-capture-binding + "REGEX" + regex-name + source + (+ start regex-rel) + (+ start regex-rel (string-length regex-name)))))] + [extra (finding-extra-for-match + rule + bindings + (substring source start end))] + [message (render-fix-template (rule-message rule) bindings)]) + (let-values ([(start-line start-col) (offset->line-col source start)] + [(end-line end-col) (offset->line-col source end)]) + (make-finding + (rule-id rule) + path + start-line + start-col + end-line + end-col + start + end + message + (rule-severity rule) + extra)))) + +(def (scan-typescript-object-type-pattern rule path source pattern) + (and (typescript-object-type-pattern? pattern) + (let ([rx (re typescript-object-type-regex)] + [len (string-length source)]) + (let loop ([start 0] [acc '()]) + (if (> start len) + (reverse acc) + (let ([match (re-search rx source start)]) + (if match + (loop (max (+ (re-match-start match) 1) + (re-match-end match)) + (cons (typescript-object-type-finding + rule + path + source + match) + acc)) + (reverse acc)))))))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -23,6 +23,7 @@ (semgrep engine rule-plan) (semgrep engine regex-support) (semgrep engine ts-decorator-scan) + (semgrep engine ts-type-scan) (semgrep engine ts-query-scan) (semgrep engine text-support) (semgrep rule parse-rule) @@ -16737,118 +16738,6 @@ (loop (cdr candidates) (if finding (cons finding acc) acc))))))) -(def (typescript-import-equals-pattern? pattern) - (and (string-find-substring pattern "import $I =") - (string-find-substring pattern "$FOO"))) - -(def (scan-typescript-import-equals-pattern rule path source pattern) - (and (typescript-import-equals-pattern? pattern) - (let ([rx (re "\\bimport[ \\t]+[A-Za-z_$][A-Za-z0-9_$]*[ \\t]*=[ \\t]*require[ \\t]*\\([^\\n)]*")] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search rx source start)]) - (if match - (let ([next (max (+ (re-match-start match) 1) - (re-match-end match))]) - (loop next - (cons (finding-from-match - rule - path - source - match) - acc))) - (reverse acc)))))))) - -(def (typescript-object-type-pattern? pattern) - (and (string-find-substring pattern "type $TYPENAME") - (string-find-substring pattern "$TYPE &") - (string-find-substring pattern "$REGEX: string"))) - -(def typescript-object-type-regex - "\\btype[ \\t]+([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*=[ \\t]*([^&;]+)[ \\t]*&[ \\t]*\\{[ \\t]*([A-Za-z_$][A-Za-z0-9_$]*)[ \\t]*:[ \\t]*string[ \\t]*\\}[ \\t]*;?") - -(def (typescript-object-type-finding rule path source match) - (let* ([typename (re-match-group match 1)] - [type-text (string-trim (re-match-group match 2))] - [regex-name (re-match-group match 3)] - [full (re-match-full match)] - [start (re-match-start match)] - [end (re-match-end match)] - [typename-rel (or (string-find-substring-from full typename 0) 0)] - [type-rel (or (string-find-substring-from - full - type-text - (+ typename-rel (string-length typename))) - typename-rel)] - [regex-rel (or (string-find-substring-from - full - regex-name - (+ type-rel (string-length type-text))) - type-rel)] - [bindings - (list - (cons "TYPENAME" - (make-regex-capture-binding - "TYPENAME" - typename - source - (+ start typename-rel) - (+ start typename-rel (string-length typename)))) - (cons "TYPE" - (make-regex-capture-binding - "TYPE" - type-text - source - (+ start type-rel) - (+ start type-rel (string-length type-text)))) - (cons "REGEX" - (make-regex-capture-binding - "REGEX" - regex-name - source - (+ start regex-rel) - (+ start regex-rel (string-length regex-name)))))] - [extra (finding-extra-for-match - rule - bindings - (substring source start end))] - [message (render-fix-template (rule-message rule) bindings)]) - (let-values ([(start-line start-col) (offset->line-col source start)] - [(end-line end-col) (offset->line-col source end)]) - (make-finding - (rule-id rule) - path - start-line - start-col - end-line - end-col - start - end - message - (rule-severity rule) - extra)))) - -(def (scan-typescript-object-type-pattern rule path source pattern) - (and (typescript-object-type-pattern? pattern) - (let ([rx (re typescript-object-type-regex)] - [len (string-length source)]) - (let loop ([start 0] [acc '()]) - (if (> start len) - (reverse acc) - (let ([match (re-search rx source start)]) - (if match - (loop (max (+ (re-match-start match) 1) - (re-match-end match)) - (cons (typescript-object-type-finding - rule - path - source - match) - acc)) - (reverse acc)))))))) - (def (javascript-imported-member-call-pattern? pattern) (let ([trimmed (string-trim pattern)]) (and (string-find-substring trimmed "var $X = $PACKAGE.$ID(")