Go structural: statement/top-level ellipsis + deep-expr rewrite
ober
596ac6416ee39786ddb629f616eb4cac20a7f3e2
--- a/lib/semgrep/match/structural.sls +++ b/lib/semgrep/match/structural.sls @@ -24,6 +24,7 @@ (define metavariable-prefix "__sg_mvar_") (define ellipsis-metavariable-prefix "__sg_mvarellipsis_") (define ellipsis-token-name "__sg_ellipsis__") + (define deep-token-name "__sg_deep__") (define internal-match-range-name "__sg_match_range") (define-record-type structural-match (fields node bindings) @@ -153,9 +154,20 @@ (values j state #f (add-string-reversed token acc) #t))))] [else (values i state #f acc #f)]))) + (define go-stmt-ellipsis-sentinel "__sg_ellip_stmt__") + (def (go-ellipsis-replacement paren-depth) + (if (> paren-depth 0) + ellipsis-token-name + go-stmt-ellipsis-sentinel)) (def (rewrite-metavariables language source) - (let ([len (string-length source)]) - (let loop ([i 0] [state 'normal] [escaped? #f] [acc '()]) + (let ([len (string-length source)] + [go? (go-language? language)]) + (let loop ([i 0] + [state 'normal] + [escaped? #f] + [paren-depth 0] + [brace-depth 0] + [acc '()]) (if (= i len) (list->string (reverse acc)) (let ([ch (string-ref source i)]) @@ -165,54 +177,97 @@ (rewrite-metavariable-at language source i state acc)]) (if rewritten? - (loop next-i next-state next-escaped? next-acc) + (loop next-i next-state next-escaped? paren-depth + brace-depth next-acc) (cond + [(and go? + (char=? ch #\<) + (< (+ i 3) len) + (char=? (string-ref source (+ i 1)) #\.) + (char=? (string-ref source (+ i 2)) #\.) + (char=? (string-ref source (+ i 3)) #\.)) + (loop (+ i 4) 'normal #f (+ paren-depth 1) + brace-depth + (add-string-reversed + (string-append deep-token-name "(") + acc))] + [(and go? + (char=? ch #\.) + (< (+ i 3) len) + (char=? (string-ref source (+ i 1)) #\.) + (char=? (string-ref source (+ i 2)) #\.) + (char=? (string-ref source (+ i 3)) #\>)) + (loop (+ i 4) 'normal #f + (max 0 (- paren-depth 1)) brace-depth + (cons #\) acc))] [(and (char=? ch #\.) (< (+ i 2) len) (char=? (string-ref source (+ i 1)) #\.) (char=? (string-ref source (+ i 2)) #\.)) - (loop - (+ i 3) - 'normal - #f + (loop (+ i 3) 'normal #f paren-depth brace-depth (add-string-reversed - (ellipsis-token language) + (if go? + (go-ellipsis-replacement paren-depth) + (ellipsis-token language)) acc))] [(char=? ch #\") - (loop (+ i 1) 'double #f (cons ch acc))] + (loop (+ i 1) 'double #f paren-depth + brace-depth (cons ch acc))] [(and (not (string=? language "json")) (char=? ch #\')) - (loop (+ i 1) 'single #f (cons ch acc))] + (loop (+ i 1) 'single #f paren-depth + brace-depth (cons ch acc))] [(and (javascript-language? language) (char=? ch #\`)) - (loop (+ i 1) 'backtick #f (cons ch acc))] + (loop (+ i 1) 'backtick #f paren-depth + brace-depth (cons ch acc))] + [(char=? ch #\() + (loop (+ i 1) 'normal #f (+ paren-depth 1) + brace-depth (cons ch acc))] + [(char=? ch #\)) + (loop (+ i 1) 'normal #f + (max 0 (- paren-depth 1)) brace-depth + (cons ch acc))] + [(char=? ch #\{) + (loop (+ i 1) 'normal #f paren-depth + (+ brace-depth 1) (cons ch acc))] + [(char=? ch #\}) + (loop (+ i 1) 'normal #f paren-depth + (max 0 (- brace-depth 1)) (cons ch acc))] [else - (loop (+ i 1) 'normal #f (cons ch acc))])))] + (loop (+ i 1) 'normal #f paren-depth + brace-depth (cons ch acc))])))] [(or (eq? state 'double) (eq? state 'single) (eq? state 'backtick)) (cond - [escaped? (loop (+ i 1) state #f (cons ch acc))] + [escaped? + (loop (+ i 1) state #f paren-depth brace-depth + (cons ch acc))] [(char=? ch #\\) - (loop (+ i 1) state #t (cons ch acc))] + (loop (+ i 1) state #t paren-depth brace-depth + (cons ch acc))] [(and (eq? state 'double) (char=? ch #\")) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth + (cons ch acc))] [(and (eq? state 'single) (char=? ch #\')) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth + (cons ch acc))] [(and (eq? state 'backtick) (char=? ch #\`)) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth + (cons ch acc))] [else (let-values ([(next-i next-state next-escaped? next-acc rewritten?) (rewrite-metavariable-at language source i state acc)]) (if rewritten? - (loop - next-i - next-state - next-escaped? - next-acc) - (loop (+ i 1) state #f (cons ch acc))))])] - [else (loop (+ i 1) 'normal #f (cons ch acc))])))))) + (loop next-i next-state next-escaped? + paren-depth brace-depth next-acc) + (loop (+ i 1) state #f paren-depth brace-depth + (cons ch acc))))])] + [else + (loop (+ i 1) 'normal #f paren-depth brace-depth + (cons ch acc))])))))) (def (metavariable-name-from-text text) (cond [(sg-string-prefix? metavariable-prefix text) @@ -476,6 +531,10 @@ (string=? text (string-append "\"" ellipsis-token-name "\"")) + (string=? text (string-append ellipsis-token-name "()")) + (string=? + text + (string-append "func " ellipsis-token-name "()")) (and (string=? (node-type n) "expression_statement") (let ([child (single-named-child n)]) (and child @@ -1121,10 +1180,32 @@ (cons (car xs) acc)))))) (define go-pattern-scaffolds (list - (cons "func __sg_scaffold__() {\n" "\n}\n") - (cons "var __sg_scaffold__ = " "\n") - (cons "" ""))) + (list + "func __sg_scaffold__() {\n" + "\n}\n" + (string-append ellipsis-token-name "()")) + (list + "" + "" + (string-append "func " ellipsis-token-name "()")) + (list "var __sg_scaffold__ = " "\n" ellipsis-token-name))) (def (utf8-length s) (bytevector-length (string->utf8 s))) + (def (string-replace-all s old new) + (let ([slen (string-length s)] [olen (string-length old)]) + (if (= olen 0) + s + (let loop ([i 0] [acc '()]) + (cond + [(> (+ i olen) slen) + (let rem ([j i] [a acc]) + (if (>= j slen) + (list->string (reverse a)) + (rem (+ j 1) (cons (string-ref s j) a))))] + [(string=? (substring s i (+ i olen)) old) + (loop + (+ i olen) + (append (reverse (string->list new)) acc))] + [else (loop (+ i 1) (cons (string-ref s i) acc))]))))) (def (go-deepest-node-containing n start end) (let child-search ([i 0]) (if (>= i (node-matchable-child-count n)) @@ -1148,8 +1229,13 @@ (if (null? scaffolds) (values #f #f) (let* ([prefix (caar scaffolds)] - [suffix (cdar scaffolds)] - [wrapped (string-append prefix trimmed suffix)] + [suffix (cadar scaffolds)] + [ellip-form (caddar scaffolds)] + [body (string-replace-all + trimmed + go-stmt-ellipsis-sentinel + ellip-form)] + [wrapped (string-append prefix body suffix)] [result (parse-target-string "go" wrapped)] [root (parse-result-root result)]) (if (or (not root) (parse-result-has-errors? result)) @@ -1158,7 +1244,7 @@ (tree-close! (parse-result-tree result)) (loop (cdr scaffolds))) (let* ([hstart (utf8-length prefix)] - [hend (+ hstart (utf8-length trimmed))] + [hend (+ hstart (utf8-length body))] [hole (and (<= (node-start-byte root) hstart) (>= (node-end-byte root) hend) (go-deepest-node-containing --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -4,10 +4,10 @@ ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "B1616180DE7038ED") ("src/semgrep/scan.ss" . "A09A5F4FE16A2036") + ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/match/structural.ss" . "8397A532700DB860") + ("src/semgrep/match/structural.ss" . "F46F2E199123036E") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/match/structural.ss +++ b/src/semgrep/match/structural.ss @@ -22,6 +22,7 @@ (define metavariable-prefix "__sg_mvar_") (define ellipsis-metavariable-prefix "__sg_mvarellipsis_") (define ellipsis-token-name "__sg_ellipsis__") +(define deep-token-name "__sg_deep__") (define internal-match-range-name "__sg_match_range") (define-record-type structural-match @@ -174,9 +175,24 @@ (values j state #f (add-string-reversed token acc) #t))))] [else (values i state #f acc #f)]))) +;; Go-specific ellipsis encoding. Go has no bare `...` statement, so the +;; placeholder must be context-appropriate and parseable. Inside parens (call / +;; argument position) it is an identifier. Outside parens it is a STATEMENT or +;; TOP-LEVEL ellipsis whose exact form (`__sg_ellipsis__()` vs +;; `func __sg_ellipsis__()`) depends on the scaffold the pattern is wrapped in, +;; so we emit a sentinel here and resolve it per-scaffold in go-parse-pattern. +(define go-stmt-ellipsis-sentinel "__sg_ellip_stmt__") + +(def (go-ellipsis-replacement paren-depth) + (if (> paren-depth 0) + ellipsis-token-name + go-stmt-ellipsis-sentinel)) + (def (rewrite-metavariables language source) - (let ([len (string-length source)]) - (let loop ([i 0] [state 'normal] [escaped? #f] [acc '()]) + (let ([len (string-length source)] + [go? (go-language? language)]) + (let loop ([i 0] [state 'normal] [escaped? #f] + [paren-depth 0] [brace-depth 0] [acc '()]) (if (= i len) (list->string (reverse acc)) (let ([ch (string-ref source i)]) @@ -185,8 +201,27 @@ (let-values ([(next-i next-state next-escaped? next-acc rewritten?) (rewrite-metavariable-at language source i state acc)]) (if rewritten? - (loop next-i next-state next-escaped? next-acc) + (loop next-i next-state next-escaped? paren-depth brace-depth next-acc) (cond + ;; Go deep-expression open `<...` -> __sg_deep__( + [(and go? + (char=? ch #\<) + (< (+ i 3) len) + (char=? (string-ref source (+ i 1)) #\.) + (char=? (string-ref source (+ i 2)) #\.) + (char=? (string-ref source (+ i 3)) #\.)) + (loop (+ i 4) 'normal #f (+ paren-depth 1) brace-depth + (add-string-reversed + (string-append deep-token-name "(") acc))] + ;; Go deep-expression close `...>` -> ) + [(and go? + (char=? ch #\.) + (< (+ i 3) len) + (char=? (string-ref source (+ i 1)) #\.) + (char=? (string-ref source (+ i 2)) #\.) + (char=? (string-ref source (+ i 3)) #\>)) + (loop (+ i 4) 'normal #f (max 0 (- paren-depth 1)) brace-depth + (cons #\) acc))] [(and (char=? ch #\.) (< (+ i 2) len) (char=? (string-ref source (+ i 1)) #\.) @@ -194,23 +229,32 @@ (loop (+ i 3) 'normal #f - (add-string-reversed (ellipsis-token language) acc))] - [(char=? ch #\") (loop (+ i 1) 'double #f (cons ch acc))] + paren-depth brace-depth + (add-string-reversed + (if go? + (go-ellipsis-replacement paren-depth) + (ellipsis-token language)) + acc))] + [(char=? ch #\") (loop (+ i 1) 'double #f paren-depth brace-depth (cons ch acc))] [(and (not (string=? language "json")) (char=? ch #\')) - (loop (+ i 1) 'single #f (cons ch acc))] + (loop (+ i 1) 'single #f paren-depth brace-depth (cons ch acc))] [(and (javascript-language? language) (char=? ch #\`)) - (loop (+ i 1) 'backtick #f (cons ch acc))] - [else (loop (+ i 1) 'normal #f (cons ch acc))])))] + (loop (+ i 1) 'backtick #f paren-depth brace-depth (cons ch acc))] + [(char=? ch #\() (loop (+ i 1) 'normal #f (+ paren-depth 1) brace-depth (cons ch acc))] + [(char=? ch #\)) (loop (+ i 1) 'normal #f (max 0 (- paren-depth 1)) brace-depth (cons ch acc))] + [(char=? ch #\{) (loop (+ i 1) 'normal #f paren-depth (+ brace-depth 1) (cons ch acc))] + [(char=? ch #\}) (loop (+ i 1) 'normal #f paren-depth (max 0 (- brace-depth 1)) (cons ch acc))] + [else (loop (+ i 1) 'normal #f paren-depth brace-depth (cons ch acc))])))] [(or (eq? state 'double) (eq? state 'single) (eq? state 'backtick)) (cond - [escaped? (loop (+ i 1) state #f (cons ch acc))] - [(char=? ch #\\) (loop (+ i 1) state #t (cons ch acc))] + [escaped? (loop (+ i 1) state #f paren-depth brace-depth (cons ch acc))] + [(char=? ch #\\) (loop (+ i 1) state #t paren-depth brace-depth (cons ch acc))] [(and (eq? state 'double) (char=? ch #\")) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth (cons ch acc))] [(and (eq? state 'single) (char=? ch #\')) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth (cons ch acc))] [(and (eq? state 'backtick) (char=? ch #\`)) - (loop (+ i 1) 'normal #f (cons ch acc))] + (loop (+ i 1) 'normal #f paren-depth brace-depth (cons ch acc))] [else (let-values ([(next-i next-state next-escaped? next-acc rewritten?) (rewrite-metavariable-at @@ -220,9 +264,9 @@ state acc)]) (if rewritten? - (loop next-i next-state next-escaped? next-acc) - (loop (+ i 1) state #f (cons ch acc))))])] - [else (loop (+ i 1) 'normal #f (cons ch acc))])))))) + (loop next-i next-state next-escaped? paren-depth brace-depth next-acc) + (loop (+ i 1) state #f paren-depth brace-depth (cons ch acc))))])] + [else (loop (+ i 1) 'normal #f paren-depth brace-depth (cons ch acc))])))))) (def (metavariable-name-from-text text) (cond @@ -504,6 +548,9 @@ (let ([text (node-text n)]) (or (string=? text ellipsis-token-name) (string=? text (string-append "\"" ellipsis-token-name "\"")) + ;; Go statement / top-level ellipsis encodings + (string=? text (string-append ellipsis-token-name "()")) + (string=? text (string-append "func " ellipsis-token-name "()")) (and (string=? (node-type n) "expression_statement") (let ([child (single-named-child n)]) (and child @@ -1159,15 +1206,35 @@ ;; valid Go, parse that, then extract the deepest node still spanning the ;; pattern's byte range. Scaffolds are tried in order; the first that parses ;; without error wins. +;; Each scaffold: (prefix suffix stmt-ellipsis-form). The third element is the +;; parseable form the statement-ellipsis sentinel resolves to in that context. (define go-pattern-scaffolds (list - (cons "func __sg_scaffold__() {\n" "\n}\n") ; statements, calls, assignments - (cons "var __sg_scaffold__ = " "\n") ; expressions (ident, selector) - (cons "" ""))) ; top-level declarations + (list "func __sg_scaffold__() {\n" "\n}\n" + (string-append ellipsis-token-name "()")) ; stmts/calls/assigns + (list "" "" + (string-append "func " ellipsis-token-name "()")) ; top-level decls + (list "var __sg_scaffold__ = " "\n" + ellipsis-token-name))) ; expressions (def (utf8-length s) (bytevector-length (string->utf8 s))) +(def (string-replace-all s old new) + (let ([slen (string-length s)] [olen (string-length old)]) + (if (= olen 0) + s + (let loop ([i 0] [acc '()]) + (cond + [(> (+ i olen) slen) + (let rem ([j i] [a acc]) + (if (>= j slen) + (list->string (reverse a)) + (rem (+ j 1) (cons (string-ref s j) a))))] + [(string=? (substring s i (+ i olen)) old) + (loop (+ i olen) (append (reverse (string->list new)) acc))] + [else (loop (+ i 1) (cons (string-ref s i) acc))]))))) + (def (go-deepest-node-containing n start end) ;; Precondition: n's byte range contains [start, end). Returns a fresh ;; node-copy of the deepest descendant whose range still contains it. @@ -1193,8 +1260,10 @@ (if (null? scaffolds) (values #f #f) (let* ([prefix (caar scaffolds)] - [suffix (cdar scaffolds)] - [wrapped (string-append prefix trimmed suffix)] + [suffix (cadar scaffolds)] + [ellip-form (caddar scaffolds)] + [body (string-replace-all trimmed go-stmt-ellipsis-sentinel ellip-form)] + [wrapped (string-append prefix body suffix)] [result (parse-target-string "go" wrapped)] [root (parse-result-root result)]) (if (or (not root) (parse-result-has-errors? result)) @@ -1203,7 +1272,7 @@ (tree-close! (parse-result-tree result)) (loop (cdr scaffolds))) (let* ([hstart (utf8-length prefix)] - [hend (+ hstart (utf8-length trimmed))] + [hend (+ hstart (utf8-length body))] [hole (and (<= (node-start-byte root) hstart) (>= (node-end-byte root) hend) (go-deepest-node-containing root hstart hend))])