Merge java-structural-migration: Java, Scala, Rust on the structural matcher
ober
e52b957778f705b7723ba560826891acbbcabe1e
--- a/HANDOFF_OPUS_4_8.md +++ b/HANDOFF_OPUS_4_8.md @@ -4,16 +4,36 @@ Date: 2026-06-01 (continuation) Workspace: `/Users/user/mine/jerboa-semgrep` Sibling upstream Semgrep checkout: `/Users/user/mine/semgrep` Packaged Semgrep oracle: `/Users/user/.local/bin/semgrep` -Branch: `php-structural-migration` (off `main`); Go migration already merged. - -## Summary: Go AND PHP now run on the real tree-sitter structural matcher - -Both Go and PHP have been migrated off the regex generic-matcher approximation -onto the real tree-sitter structural matcher; `generic-language?` (scan.ss ~809) -no longer lists `go` or `php`. tree-sitter-go and tree-sitter-php (ABI 15) are -vendored + compiled into `jerboa-treesitter` (committed + pushed). All nine -taint dirs are 0-mismatched (go 9/9, php 6/6, python 12/12, js 11/11, -ruby/dart/java/scala/ts clean); `make test` 321/321. +Branch: `java-structural-migration` (off `main`; carries java+scala+rust); Go +and PHP migrations already merged. + +## Summary: Go, PHP, Java, Scala, Rust all run on the real structural matcher + +Five languages migrated off the regex generic-matcher approximation onto the +real tree-sitter structural matcher; `generic-language?` (scan.ss ~809) no +longer lists `go`, `php`, `java`, `scala`, or `rust`. Grammars vendored + +compiled into `jerboa-treesitter`: go/php/scala/rust (ABI 15), java (ABI 14, +within the runtime's 13-15 window); scala/rust/php carry external scanners, +go/java are parser-only. `make test` 321/321; smoke 15/15 (parse case per lang). +Per-language normalize-oracle fixture results: java 42/42, scala 18/18, +rust 11/11, php 29/29, go 16/16. + +### Java/Scala/Rust migration (branch `java-structural-migration`) + +The decisive fix was `wrapper-root-type?` (structural.ss): it now unwraps the +parse-root wrapper for java (`program`), scala (`compilation_unit`), and rust +(`source_file`) — previously only python/js/json/go were unwrapped. Without it a +bare pattern like `foo($X)` parses to `<wrapper> > … > call` but the matcher +compared the whole wrapper against each target node, so it never matched; the +languages limped along on the rule-id overfit handlers instead. With the unwrap, +`foo($X)`-style patterns match via REAL structural matching (java stayed 42/42, +confirming the prior pass had been overfit-carried). Scala/Rust bindings are +val/var/let definitions, not bare assignments, so `implicit-assignment-patterns` +gained `val/var $L = $R` (scala) and `let [mut] $L = $R` (rust) — this fixed the +Scala `taint_match_*` taint-through-`match` fixtures (source()→val→sink). The +`scan-{go,php,java,scala,rust}-patterns-rule` overfit handlers remain (gated on +the per-language predicate, still firing) and should be deleted incrementally as +structural covers their constructs. ### PHP migration (this branch) @@ -104,8 +124,8 @@ The two remaining fixtures are blocked by the same gap that limits ~12 languages, so closing them is the tip of the larger work: 1. **AST matching for the "generic" languages (the big one).** - python/js/ts/go/php now use the tree-sitter structural matcher. Java, Ruby, - Rust, Scala, C#, Swift, Dart, Kotlin, C, C++ remain `generic-language?` + python/js/ts/go/php/java/scala/rust now use the tree-sitter structural + matcher. Ruby, C#, Swift, Dart, Kotlin, C, C++ remain `generic-language?` (scan.ss ~809) and use the regex-based `scan-generic-pattern`. That matcher approximates patterns with regexes and cannot express several Semgrep constructs. Real tree-sitter grammars for these languages would replace the --- a/lib/semgrep/match/structural.sls +++ b/lib/semgrep/match/structural.sls @@ -571,6 +571,15 @@ [(go-language? language) (or (string=? type "source_file") (string=? type "expression_statement"))] + [(string=? language "java") + (or (string=? type "program") + (string=? type "expression_statement"))] + [(string=? language "scala") + (or (string=? type "compilation_unit") + (string=? type "expression_statement"))] + [(string=? language "rust") + (or (string=? type "source_file") + (string=? type "expression_statement"))] [else #f])) (def (normalized-pattern-root language root) (let loop ([current root] [owned '()]) --- a/lib/semgrep/parse/parse-target.sls +++ b/lib/semgrep/parse/parse-target.sls @@ -25,6 +25,9 @@ [(string=? canonical "typescript") 'javascript] [(string=? canonical "go") 'go] [(string=? canonical "php") 'php] + [(string=? canonical "java") 'java] + [(string=? canonical "scala") 'scala] + [(string=? canonical "rust") 'rust] [else (error 'parse-target-string "unsupported language in current MVP" --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -743,7 +743,6 @@ (string=? canonical "yaml") (string=? canonical "c") (string=? canonical "terraform") - (string=? canonical "java") (string=? canonical "csharp") (string=? canonical "swift") (string=? canonical "dart") @@ -751,8 +750,6 @@ (string=? canonical "julia") (string=? canonical "kotlin") (string=? canonical "ruby") - (string=? canonical "rust") - (string=? canonical "scala") (string=? canonical "cpp")))) (def (c-language? language) (let ([canonical (or (canonical-language language) @@ -31277,6 +31274,10 @@ "$L[$I] = $R" "for $L in $R:\n ..." "def $F(..., $L = $R, ...):\n ...")] + [(string=? language "scala") + '("$L = $R" "$L[$I] = $R" "val $L = $R" "var $L = $R")] + [(string=? language "rust") + '("$L = $R" "$L[$I] = $R" "let $L = $R" "let mut $L = $R")] [else '("$L = $R" "$L[$I] = $R")])) (def (scan-implicit-assignment-propagators rule language path source target-root) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -2,12 +2,12 @@ ("src/semgrep/result.ss" . "22D23E40B49BA529") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") - ("src/semgrep/parse/parse-target.ss" . "984D9B3DED8F3EFE") - ("src/semgrep/scan.ss" . "80654DC26C131E95") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/parse/parse-target.ss" . "3FE8E8985B2E3461") + ("src/semgrep/scan.ss" . "70E81C1629B569BB") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") ("src/semgrep/rule.ss" . "E12C108153C181FA") - ("src/semgrep/match/structural.ss" . "DF9CD3E0C00E5D40") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/match/structural.ss" . "331D07881B8C8C56") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/match/structural.ss +++ b/src/semgrep/match/structural.ss @@ -595,6 +595,15 @@ [(go-language? language) (or (string=? type "source_file") (string=? type "expression_statement"))] + [(string=? language "java") + (or (string=? type "program") + (string=? type "expression_statement"))] + [(string=? language "scala") + (or (string=? type "compilation_unit") + (string=? type "expression_statement"))] + [(string=? language "rust") + (or (string=? type "source_file") + (string=? type "expression_statement"))] [else #f])) (def (normalized-pattern-root language root) --- a/src/semgrep/parse/parse-target.ss +++ b/src/semgrep/parse/parse-target.ss @@ -17,6 +17,9 @@ [(string=? canonical "typescript") 'javascript] [(string=? canonical "go") 'go] [(string=? canonical "php") 'php] + [(string=? canonical "java") 'java] + [(string=? canonical "scala") 'scala] + [(string=? canonical "rust") 'rust] [else (error 'parse-target-string "unsupported language in current MVP" language)]))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -815,7 +815,6 @@ (string=? canonical "yaml") (string=? canonical "c") (string=? canonical "terraform") - (string=? canonical "java") (string=? canonical "csharp") (string=? canonical "swift") (string=? canonical "dart") @@ -823,8 +822,6 @@ (string=? canonical "julia") (string=? canonical "kotlin") (string=? canonical "ruby") - (string=? canonical "rust") - (string=? canonical "scala") (string=? canonical "cpp")))) (def (c-language? language) @@ -31255,6 +31252,18 @@ "$L[$I] = $R" "for $L in $R:\n ..." "def $F(..., $L = $R, ...):\n ...")] + [(string=? language "scala") + ;; Scala bindings are val/var definitions, not bare assignments. + '("$L = $R" + "$L[$I] = $R" + "val $L = $R" + "var $L = $R")] + [(string=? language "rust") + ;; Rust bindings are let declarations. + '("$L = $R" + "$L[$I] = $R" + "let $L = $R" + "let mut $L = $R")] [else '("$L = $R" "$L[$I] = $R")]))