Route Dart through the structural matcher (scaffolding + semicolon trim)
ober
3f9393f8ea0a5ba889267f9585b16fa702645966
--- a/HANDOFF_OPUS_4_8.md +++ b/HANDOFF_OPUS_4_8.md @@ -7,7 +7,7 @@ Packaged Semgrep oracle: `/Users/user/.local/bin/semgrep` Branch: `java-structural-migration` (off `main`; carries java+scala+rust); Go and PHP migrations already merged. -## Summary: Go, PHP, Java, Scala, Rust, Ruby, Kotlin on the structural matcher +## Summary: Go, PHP, Java, Scala, Rust, Ruby, Kotlin, Dart on structural Five languages migrated off the regex generic-matcher approximation onto the real tree-sitter structural matcher; `generic-language?` (scan.ss ~809) no @@ -16,7 +16,7 @@ compiled into `jerboa-treesitter`: go/php/scala/rust (ABI 15), java (ABI 14, within the runtime's 13-15 window); scala/rust/php carry external scanners, go/java are parser-only. `make test` 321/321; smoke 15/15 (parse case per lang). Per-language normalize-oracle fixture results: java 42/42, scala 18/18, -rust 11/11, ruby 4/4, kotlin 4/4, php 29/29, go 16/16. +rust 11/11, ruby 4/4, kotlin 4/4, dart 5/5, php 29/29, go 16/16. ### Java/Scala/Rust migration (branch `java-structural-migration`) @@ -124,10 +124,11 @@ The two remaining fixtures are blocked by the same gap that limits ~12 languages, so closing them is the tip of the larger work: 1. **AST matching for the "generic" languages (the big one).** - python/js/ts/go/php/java/scala/rust/ruby/kotlin now use the tree-sitter - structural matcher. C#, Swift, Dart, C, C++ remain (Dart grammar is vendored - but deferred: bare patterns parse as top-level declarations, needing Go-style - scaffolding) `generic-language?` + python/js/ts/go/php/java/scala/rust/ruby/kotlin/dart now use the tree-sitter + structural matcher. C#, Swift, C, C++ remain. Dart needed Go-style pattern + scaffolding (bare patterns parse as top-level declarations) plus a `;`-trim + (dart has no wrapping call node, so an expression pattern matches the whole + expression_statement; maybe-trim-dart-statement-semicolon drops the terminator) `generic-language?` (scan.ss ~809) and use the regex-based `scan-generic-pattern`. That matcher approximates patterns with regexes and cannot express several Semgrep constructs. Real tree-sitter grammars for these languages would replace the --- a/lib/semgrep/match/structural.sls +++ b/lib/semgrep/match/structural.sls @@ -1650,6 +1650,40 @@ (begin (tree-close! (parse-result-tree result)) (loop (cdr scaffolds))))))))))) + (define dart-pattern-scaffolds + (list + (cons "class _C{void _m(){" "}}") + (cons "class _C{void _m(){" ";}}") + (cons "" ""))) + (def (dart-parse-pattern rewritten-pattern) + (let ([trimmed (string-trim rewritten-pattern)]) + (let loop ([scaffolds dart-pattern-scaffolds]) + (if (null? scaffolds) + (values #f #f) + (let* ([prefix (caar scaffolds)] + [suffix (cdar scaffolds)] + [wrapped (string-append prefix trimmed suffix)] + [result (parse-target-string "dart" wrapped)] + [root (parse-result-root result)]) + (if (or (not root) (parse-result-has-errors? result)) + (begin + (when root (node-close! root)) + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))) + (let* ([hstart (utf8-length prefix)] + [hend (+ hstart (utf8-length trimmed))] + [hole (and (<= (node-start-byte root) hstart) + (>= (node-end-byte root) hend) + (go-deepest-node-containing + root + hstart + hend))]) + (node-close! root) + (if hole + (values hole result) + (begin + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))))))))))) (def (run-structural-matches language normalized @@ -1688,11 +1722,17 @@ language pattern-source)]) (cond - [(or (go-language? language) (php-language? language)) + [(or (go-language? language) + (php-language? language) + (string=? language "dart")) (let-values ([(normalized result) - (if (go-language? language) - (go-parse-pattern rewritten-pattern) - (php-parse-pattern rewritten-pattern))]) + (cond + [(go-language? language) + (go-parse-pattern rewritten-pattern)] + [(php-language? language) + (php-parse-pattern rewritten-pattern)] + [else + (dart-parse-pattern rewritten-pattern)])]) (if (not normalized) '() (let ([matches (run-structural-matches --- a/lib/semgrep/parse/parse-target.sls +++ b/lib/semgrep/parse/parse-target.sls @@ -30,6 +30,7 @@ [(string=? canonical "rust") 'rust] [(string=? canonical "ruby") 'ruby] [(string=? canonical "kotlin") 'kotlin] + [(string=? canonical "dart") 'dart] [else (error 'parse-target-string "unsupported language in current MVP" --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -488,6 +488,22 @@ (not (semicolon-trimmable-node? node))) (extend-finding-over-semicolon rule finding source) finding)) + (def (maybe-trim-dart-statement-semicolon language pattern + source node finding) + (if (and (string=? language "dart") + (not (pattern-trailing-semicolon? pattern)) + (string=? (node-type node) "expression_statement") + (let ([t (node-text node)]) + (and (> (string-length t) 0) + (char=? + (string-ref t (- (string-length t) 1)) + #\;)))) + (finding-with-range + finding + source + (finding-start-offset finding) + (- (finding-end-offset finding) 1)) + finding)) (def (base-finding-extra rule) (if (null? (rule-metadata rule)) '() @@ -745,7 +761,6 @@ (string=? canonical "terraform") (string=? canonical "csharp") (string=? canonical "swift") - (string=? canonical "dart") (string=? canonical "move_on_aptos") (string=? canonical "julia") (string=? canonical "cpp")))) @@ -20311,8 +20326,9 @@ finding0)] [finding (if range-binding finding1 - (maybe-extend-expression-semicolon rule pattern source node - finding1))] + (maybe-trim-dart-statement-semicolon language pattern source node + (maybe-extend-expression-semicolon rule pattern source node + finding1)))] [focused (apply-rule-focus rule finding)]) (node-close! node) (loop (cdr xs) (append (reverse focused) acc)))))))) @@ -31278,6 +31294,12 @@ '("$L = $R" "$L[$I] = $R" "let $L = $R" "let mut $L = $R")] [(string=? language "kotlin") '("$L = $R" "$L[$I] = $R" "val $L = $R" "var $L = $R")] + [(string=? language "dart") + '("$L = $R" + "$L[$I] = $R" + "var $L = $R" + "final $L = $R" + "$T $L = $R")] [else '("$L = $R" "$L[$I] = $R")])) (def (scan-implicit-assignment-propagators rule language path source target-root) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -2,12 +2,12 @@ ("src/semgrep/result.ss" . "22D23E40B49BA529") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") - ("src/semgrep/parse/parse-target.ss" . "904B71D207617229") - ("src/semgrep/scan.ss" . "A994804D70D4F26E") - ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") + ("src/semgrep/scan.ss" . "7E23CE2332C489AB") ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") - ("src/semgrep/match/structural.ss" . "6047821D83B32FCE") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/match/structural.ss" . "391A295E5733B0B0") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/match/structural.ss +++ b/src/semgrep/match/structural.ss @@ -1647,6 +1647,41 @@ (tree-close! (parse-result-tree result)) (loop (cdr scaffolds))))))))))) +;; Dart bare patterns parse as top-level declarations (`sink($X)` becomes a +;; function_signature), so we wrap them in a class method body and extract the +;; spanned node by byte range, like the PHP/Go scaffolds. +(define dart-pattern-scaffolds + (list (cons "class _C{void _m(){" "}}") ; statements + (cons "class _C{void _m(){" ";}}") ; bare expressions (need a `;`) + (cons "" ""))) ; top-level declarations + +(def (dart-parse-pattern rewritten-pattern) + (let ([trimmed (string-trim rewritten-pattern)]) + (let loop ([scaffolds dart-pattern-scaffolds]) + (if (null? scaffolds) + (values #f #f) + (let* ([prefix (caar scaffolds)] + [suffix (cdar scaffolds)] + [wrapped (string-append prefix trimmed suffix)] + [result (parse-target-string "dart" wrapped)] + [root (parse-result-root result)]) + (if (or (not root) (parse-result-has-errors? result)) + (begin + (when root (node-close! root)) + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))) + (let* ([hstart (utf8-length prefix)] + [hend (+ hstart (utf8-length trimmed))] + [hole (and (<= (node-start-byte root) hstart) + (>= (node-end-byte root) hend) + (go-deepest-node-containing root hstart hend))]) + (node-close! root) + (if hole + (values hole result) + (begin + (tree-close! (parse-result-tree result)) + (loop (cdr scaffolds))))))))))) + (def (run-structural-matches language normalized target-root initial-bindings) (let* ([chain-ellipsis? (chain-node-has-ellipsis? normalized)] [raw-matches @@ -1680,11 +1715,14 @@ initial-bindings) (let ([rewritten-pattern (rewrite-metavariables language pattern-source)]) (cond - [(or (go-language? language) (php-language? language)) + [(or (go-language? language) + (php-language? language) + (string=? language "dart")) (let-values ([(normalized result) - (if (go-language? language) - (go-parse-pattern rewritten-pattern) - (php-parse-pattern rewritten-pattern))]) + (cond + [(go-language? language) (go-parse-pattern rewritten-pattern)] + [(php-language? language) (php-parse-pattern rewritten-pattern)] + [else (dart-parse-pattern rewritten-pattern)])]) (if (not normalized) '() (let ([matches (run-structural-matches --- a/src/semgrep/parse/parse-target.ss +++ b/src/semgrep/parse/parse-target.ss @@ -22,6 +22,7 @@ [(string=? canonical "rust") 'rust] [(string=? canonical "ruby") 'ruby] [(string=? canonical "kotlin") 'kotlin] + [(string=? canonical "dart") 'dart] [else (error 'parse-target-string "unsupported language in current MVP" language)]))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -554,6 +554,24 @@ (extend-finding-over-semicolon rule finding source) finding)) +;; Dart has no wrapping expression node for a call (`print(x)` is an identifier +;; plus a selector under expression_statement), so an expression pattern like +;; `print($X)` extracts/matches the whole statement, including its `;`. When the +;; pattern itself has no trailing `;`, trim that statement terminator so the +;; finding range matches Semgrep's expression span. +(def (maybe-trim-dart-statement-semicolon language pattern source node finding) + (if (and (string=? language "dart") + (not (pattern-trailing-semicolon? pattern)) + (string=? (node-type node) "expression_statement") + (let ([t (node-text node)]) + (and (> (string-length t) 0) + (char=? (string-ref t (- (string-length t) 1)) #\;)))) + (finding-with-range finding + source + (finding-start-offset finding) + (- (finding-end-offset finding) 1)) + finding)) + (def (base-finding-extra rule) (if (null? (rule-metadata rule)) '() @@ -817,7 +835,6 @@ (string=? canonical "terraform") (string=? canonical "csharp") (string=? canonical "swift") - (string=? canonical "dart") (string=? canonical "move_on_aptos") (string=? canonical "julia") (string=? canonical "cpp")))) @@ -20157,12 +20174,17 @@ finding0)] [finding (if range-binding finding1 - (maybe-extend-expression-semicolon - rule + (maybe-trim-dart-statement-semicolon + language pattern source node - finding1))] + (maybe-extend-expression-semicolon + rule + pattern + source + node + finding1)))] [focused (apply-rule-focus rule finding)]) (node-close! node) (loop (cdr xs) (append (reverse focused) acc)))))))) @@ -31268,6 +31290,13 @@ "$L[$I] = $R" "val $L = $R" "var $L = $R")] + [(string=? language "dart") + ;; Dart bindings: var/final/typed local declarations. + '("$L = $R" + "$L[$I] = $R" + "var $L = $R" + "final $L = $R" + "$T $L = $R")] [else '("$L = $R" "$L[$I] = $R")]))