Route Go through the tree-sitter structural matcher
ober
abf9715fc308158b64be51fc0571d4e8bb99de33
--- a/lib/semgrep/match/structural.sls +++ b/lib/semgrep/match/structural.sls @@ -44,6 +44,7 @@ (string=? language "typescript") (string=? language "ts") (string=? language "tsx"))) + (def (go-language? language) (string=? language "go")) (def (sg-string-prefix? prefix s) (let ([prefix-len (string-length prefix)] [len (string-length s)]) @@ -444,6 +445,9 @@ (or (string=? type "program") (string=? type "expression_statement"))] [(string=? language "json") (string=? type "document")] + [(go-language? language) + (or (string=? type "source_file") + (string=? type "expression_statement"))] [else #f])) (def (normalized-pattern-root language root) (let loop ([current root] [owned '()]) @@ -1115,50 +1119,125 @@ (inner (cdr ys))))) acc (cons (car xs) acc)))))) + (define go-pattern-scaffolds + (list + (cons "func __sg_scaffold__() {\n" "\n}\n") + (cons "var __sg_scaffold__ = " "\n") + (cons "" ""))) + (def (utf8-length s) (bytevector-length (string->utf8 s))) + (def (go-deepest-node-containing n start end) + (let child-search ([i 0]) + (if (>= i (node-matchable-child-count n)) + (node-copy n) + (let ([child (node-matchable-child n i)]) + (if (and child + (<= (node-start-byte child) start) + (>= (node-end-byte child) end)) + (let ([result (go-deepest-node-containing + child + start + end)]) + (node-close! child) + result) + (begin + (when child (node-close! child)) + (child-search (+ i 1)))))))) + (def (go-parse-pattern rewritten-pattern) + (let ([trimmed (string-trim rewritten-pattern)]) + (let loop ([scaffolds go-pattern-scaffolds]) + (if (null? scaffolds) + (values #f #f) + (let* ([prefix (caar scaffolds)] + [suffix (cdar scaffolds)] + [wrapped (string-append prefix trimmed suffix)] + [result (parse-target-string "go" wrapped)] + [root (parse-result-root result)]) + (if (or (not root) (parse-result-has-errors? result)) + (begin + (when root (node-close! root)) + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))) + (let* ([hstart (utf8-length prefix)] + [hend (+ hstart (utf8-length trimmed))] + [hole (and (<= (node-start-byte root) hstart) + (>= (node-end-byte root) hend) + (go-deepest-node-containing + root + hstart + hend))]) + (node-close! root) + (if hole + (values hole result) + (begin + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))))))))))) + (def (run-structural-matches + language + normalized + target-root + initial-bindings) + (let* ([chain-ellipsis? (chain-node-has-ellipsis? + normalized)] + [raw-matches (reverse + (walk-target + target-root + (lambda (candidate acc) + (let ([matches (structural-node-matches + language + normalized + candidate + initial-bindings)]) + (let loop ([xs matches] [match-acc acc]) + (if (null? xs) + match-acc + (loop + (cdr xs) + (cons + (make-structural-match + (node-copy candidate) + (car xs)) + match-acc))))))))]) + (if chain-ellipsis? + (drop-contained-structural-matches raw-matches) + raw-matches))) (def (structural-pattern-matches-with-bindings language pattern-source target-root initial-bindings) - (let* ([rewritten-pattern (rewrite-metavariables - language - pattern-source)] - [pattern-result (parse-target-string - language - rewritten-pattern)] - [pattern-root (parse-result-root pattern-result)]) - (let-values ([(normalized owned-nodes) - (normalized-pattern-root language pattern-root)]) - (let* ([chain-ellipsis? (chain-node-has-ellipsis? - normalized)] - [raw-matches (reverse - (walk-target - target-root - (lambda (candidate acc) - (let ([matches (structural-node-matches - language - normalized - candidate - initial-bindings)]) - (let loop ([xs matches] - [match-acc acc]) - (if (null? xs) - match-acc - (loop - (cdr xs) - (cons - (make-structural-match - (node-copy candidate) - (car xs)) - match-acc))))))))] - [matches (if chain-ellipsis? - (drop-contained-structural-matches - raw-matches) - raw-matches)]) - (for-each node-close! owned-nodes) - (when pattern-root (node-close! pattern-root)) - (tree-close! (parse-result-tree pattern-result)) - matches)))) + (let ([rewritten-pattern (rewrite-metavariables + language + pattern-source)]) + (if (go-language? language) + (let-values ([(normalized result) + (go-parse-pattern rewritten-pattern)]) + (if (not normalized) + '() + (let ([matches (run-structural-matches + language + normalized + target-root + initial-bindings)]) + (node-close! normalized) + (tree-close! (parse-result-tree result)) + matches))) + (let* ([pattern-result (parse-target-string + language + rewritten-pattern)] + [pattern-root (parse-result-root pattern-result)]) + (let-values ([(normalized owned-nodes) + (normalized-pattern-root + language + pattern-root)]) + (let ([matches (run-structural-matches + language + normalized + target-root + initial-bindings)]) + (for-each node-close! owned-nodes) + (when pattern-root (node-close! pattern-root)) + (tree-close! (parse-result-tree pattern-result)) + matches)))))) (def (structural-pattern-matches language pattern-source --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -753,7 +753,6 @@ (string=? canonical "kotlin") (string=? canonical "ruby") (string=? canonical "rust") - (string=? canonical "go") (string=? canonical "scala") (string=? canonical "cpp")))) (def (c-language? language) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -3,11 +3,11 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "B1616180DE7038ED") - ("src/semgrep/scan.ss" . "621FB6C8F6C1151A") + ("src/semgrep/scan.ss" . "A09A5F4FE16A2036") ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/match/structural.ss" . "6FE77014EE9FDCE4") + ("src/semgrep/match/structural.ss" . "8397A532700DB860") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/match/structural.ss +++ b/src/semgrep/match/structural.ss @@ -53,6 +53,9 @@ (string=? language "ts") (string=? language "tsx"))) +(def (go-language? language) + (string=? language "go")) + (def (sg-string-prefix? prefix s) (let ([prefix-len (string-length prefix)] [len (string-length s)]) @@ -469,6 +472,9 @@ (string=? type "expression_statement"))] [(string=? language "json") (string=? type "document")] + [(go-language? language) + (or (string=? type "source_file") + (string=? type "expression_statement"))] [else #f])) (def (normalized-pattern-root language root) @@ -1146,44 +1152,119 @@ acc (cons (car xs) acc)))))) +;; --------------------------------------------------------------------------- +;; Go pattern parsing. Go's top-level grammar only accepts declarations, so a +;; bare expression/statement pattern (`sink(...)`, `reader.File`, `source`) +;; cannot be parsed directly. We wrap the pattern in a scaffold that makes it +;; valid Go, parse that, then extract the deepest node still spanning the +;; pattern's byte range. Scaffolds are tried in order; the first that parses +;; without error wins. +(define go-pattern-scaffolds + (list + (cons "func __sg_scaffold__() {\n" "\n}\n") ; statements, calls, assignments + (cons "var __sg_scaffold__ = " "\n") ; expressions (ident, selector) + (cons "" ""))) ; top-level declarations + +(def (utf8-length s) + (bytevector-length (string->utf8 s))) + +(def (go-deepest-node-containing n start end) + ;; Precondition: n's byte range contains [start, end). Returns a fresh + ;; node-copy of the deepest descendant whose range still contains it. + (let child-search ([i 0]) + (if (>= i (node-matchable-child-count n)) + (node-copy n) + (let ([child (node-matchable-child n i)]) + (if (and child + (<= (node-start-byte child) start) + (>= (node-end-byte child) end)) + (let ([result (go-deepest-node-containing child start end)]) + (node-close! child) + result) + (begin + (when child (node-close! child)) + (child-search (+ i 1)))))))) + +(def (go-parse-pattern rewritten-pattern) + ;; Returns (values pattern-node parse-result) on success, or (values #f #f). + ;; Caller must node-close! the node and tree-close! the parse-result tree. + (let ([trimmed (string-trim rewritten-pattern)]) + (let loop ([scaffolds go-pattern-scaffolds]) + (if (null? scaffolds) + (values #f #f) + (let* ([prefix (caar scaffolds)] + [suffix (cdar scaffolds)] + [wrapped (string-append prefix trimmed suffix)] + [result (parse-target-string "go" wrapped)] + [root (parse-result-root result)]) + (if (or (not root) (parse-result-has-errors? result)) + (begin + (when root (node-close! root)) + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))) + (let* ([hstart (utf8-length prefix)] + [hend (+ hstart (utf8-length trimmed))] + [hole (and (<= (node-start-byte root) hstart) + (>= (node-end-byte root) hend) + (go-deepest-node-containing root hstart hend))]) + (node-close! root) + (if hole + (values hole result) + (begin + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))))))))))) + +(def (run-structural-matches language normalized target-root initial-bindings) + (let* ([chain-ellipsis? (chain-node-has-ellipsis? normalized)] + [raw-matches + (reverse + (walk-target + target-root + (lambda (candidate acc) + (let ([matches + (structural-node-matches + language + normalized + candidate + initial-bindings)]) + (let loop ([xs matches] [match-acc acc]) + (if (null? xs) + match-acc + (loop + (cdr xs) + (cons (make-structural-match + (node-copy candidate) + (car xs)) + match-acc))))))))]) + (if chain-ellipsis? + (drop-contained-structural-matches raw-matches) + raw-matches))) + (def (structural-pattern-matches-with-bindings language pattern-source target-root initial-bindings) - (let* ([rewritten-pattern (rewrite-metavariables language pattern-source)] - [pattern-result (parse-target-string language rewritten-pattern)] - [pattern-root (parse-result-root pattern-result)]) - (let-values ([(normalized owned-nodes) - (normalized-pattern-root language pattern-root)]) - (let* ([chain-ellipsis? (chain-node-has-ellipsis? normalized)] - [raw-matches - (reverse - (walk-target - target-root - (lambda (candidate acc) - (let ([matches - (structural-node-matches - language - normalized - candidate - initial-bindings)]) - (let loop ([xs matches] [match-acc acc]) - (if (null? xs) - match-acc - (loop - (cdr xs) - (cons (make-structural-match - (node-copy candidate) - (car xs)) - match-acc))))))))] - [matches (if chain-ellipsis? - (drop-contained-structural-matches raw-matches) - raw-matches)]) - (for-each node-close! owned-nodes) - (when pattern-root (node-close! pattern-root)) - (tree-close! (parse-result-tree pattern-result)) - matches)))) + (let ([rewritten-pattern (rewrite-metavariables language pattern-source)]) + (if (go-language? language) + (let-values ([(normalized result) (go-parse-pattern rewritten-pattern)]) + (if (not normalized) + '() + (let ([matches (run-structural-matches + language normalized target-root initial-bindings)]) + (node-close! normalized) + (tree-close! (parse-result-tree result)) + matches))) + (let* ([pattern-result (parse-target-string language rewritten-pattern)] + [pattern-root (parse-result-root pattern-result)]) + (let-values ([(normalized owned-nodes) + (normalized-pattern-root language pattern-root)]) + (let ([matches (run-structural-matches + language normalized target-root initial-bindings)]) + (for-each node-close! owned-nodes) + (when pattern-root (node-close! pattern-root)) + (tree-close! (parse-result-tree pattern-result)) + matches)))))) (def (structural-pattern-matches language pattern-source target-root) (structural-pattern-matches-with-bindings --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -825,7 +825,6 @@ (string=? canonical "kotlin") (string=? canonical "ruby") (string=? canonical "rust") - (string=? canonical "go") (string=? canonical "scala") (string=? canonical "cpp"))))