Replace 2 Go type overfit handlers with real type inference
ober
33e377eac193f19f2387d9f1bbba18845836a579
--- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -3099,7 +3099,8 @@ [else (let* ([constants (constant-bindings-before source - before-offset)] + before-offset + #f)] [value (comparison-value trimmed constants #f #f)]) (and (not (comparison-missing? value)) (comparison-value->string value)))]))) @@ -26699,20 +26700,12 @@ (scan-go-int-binop-rule rule path source)] [(go-rule-id? rule "reinstantiated_variable_in_new_block") (scan-go-reinstantiated-var-rule rule path source)] - [(go-rule-id? rule "metavar-type-func-param") - (scan-go-receiver-foo-rule - rule - path - source - "metavar-type-func-param")] [(go-rule-id? rule "typed-metavar-metavar-regex") (scan-go-receiver-foo-rule rule path source "typed-metavar-metavar-regex")] - [(go-rule-id? rule "no-direct-db-exec") - (scan-go-db-exec-rule rule path source)] [(go-rule-id? rule "wrong-err-check") (scan-go-wrong-err-check-rule rule path source)] [(go-rule-id? rule "match") @@ -28527,6 +28520,32 @@ (if (member type-name acc) acc (cons type-name acc)))) '())) '()))) + (def (go-declaration-types-before-binding binding source) + (let ([name (simple-binding-identifier binding)]) + (if name + (let* ([limit (min (metavariable-binding-start-byte binding) + (string-length source))] + [prefix (substring source 0 limit)] + [pattern (string-append + "(^|[^A-Za-z0-9_$])" + (regex-escape-string name) + "[ \\t]+" + "(\\*?(?:\\[\\])?[A-Za-z_][A-Za-z0-9_.]*)")]) + (regex-fold-matches + pattern + prefix + (lambda (match acc) + (let ([type-name (re-match-group match 2)]) + (if (or (member type-name acc) + (not (or (string-find-substring type-name "*") + (string-find-substring type-name ".") + (string-find-substring + type-name + "[")))) + acc + (cons type-name acc)))) + '())) + '()))) (def (inferred-binding-types binding) (let* ([text (string-trim (metavariable-binding-text binding))] @@ -28572,14 +28591,16 @@ candidate metavariable))]) (and binding - (let ([actual-types (append - (inferred-binding-types binding) + (let ([actual-types (append (inferred-binding-types binding) (annotation-types-before-binding binding source) (simple-declaration-types-before-binding binding source) + (go-declaration-types-before-binding + binding + source) (c-array-declaration-types-before-binding binding source))]) @@ -28954,12 +28975,19 @@ [else (strip-delimiter-pair (string-trim text))]))) (def python-simple-assignment-regex "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") + (def go-simple-assignment-regex + "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*(?::=|=)[ \\t]*([^\\n#=][^\\n#]*)") (def (constant-binding-from-value name value source offset) (make-regex-capture-binding name (comparison-value->string value) source offset offset)) - (def (constant-bindings-before source before-offset) + (def (constant-bindings-before + source + before-offset + language) (regex-fold-matches - python-simple-assignment-regex + (if (and language (go-language? language)) + go-simple-assignment-regex + python-simple-assignment-regex) (substring source 0 before-offset) (lambda (match acc) (let* ([name (re-match-group match 2)] @@ -29006,10 +29034,14 @@ (metavariable-binding-start-col binding) (metavariable-binding-end-line binding) (metavariable-binding-end-col binding))))) - (def (comparison-bindings-with-constants candidate source) + (def (comparison-bindings-with-constants + candidate + source + language) (let* ([constants (constant-bindings-before source - (finding-start-offset candidate))] + (finding-start-offset candidate) + language)] [metavars (finding-metavars candidate)] [resolved (map (lambda (entry) (cons @@ -29505,7 +29537,8 @@ (def (metavariable-comparison-satisfied? candidate clause - source) + source + language) (let* ([metavariable (alist-ref/default clause 'metavariable @@ -29515,7 +29548,8 @@ [base (alist-ref/default clause 'base #f)] [bindings (comparison-bindings-with-constants candidate - source)]) + source + language)]) (and comparison (not (finding-has-resolved-decomposition? candidate)) (or (not metavariable) @@ -29849,7 +29883,8 @@ (and (metavariable-comparison-satisfied? candidate (cdr clause) - source) + source + language) candidate)] [(focus-metavariable) candidate] [else --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -3,11 +3,11 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "B1616180DE7038ED") - ("src/semgrep/scan.ss" . "3E61EC4E4DEE7168") + ("src/semgrep/scan.ss" . "60922ED6A9C2992B") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/rule.ss" . "E12C108153C181FA") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/match/structural.ss" . "AA1BA746924C8189") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -3164,7 +3164,7 @@ [(python-fstring-literal? trimmed) (python-constant-string-literal-value trimmed source before-offset)] [else - (let* ([constants (constant-bindings-before source before-offset)] + (let* ([constants (constant-bindings-before source before-offset #f)] [value (comparison-value trimmed constants #f #f)]) (and (not (comparison-missing? value)) (comparison-value->string value)))]))) @@ -26679,13 +26679,12 @@ (scan-go-int-binop-rule rule path source)] [(go-rule-id? rule "reinstantiated_variable_in_new_block") (scan-go-reinstantiated-var-rule rule path source)] - [(go-rule-id? rule "metavar-type-func-param") - (scan-go-receiver-foo-rule rule path source "metavar-type-func-param")] + ;; metavar-type-func-param + no-direct-db-exec are now handled by the + ;; structural matcher + real Go `name Type` inference (go-declaration- + ;; types-before-binding), so their fixture-keyed handlers were removed. [(go-rule-id? rule "typed-metavar-metavar-regex") (scan-go-receiver-foo-rule rule path source "typed-metavar-metavar-regex")] - [(go-rule-id? rule "no-direct-db-exec") - (scan-go-db-exec-rule rule path source)] [(go-rule-id? rule "wrong-err-check") (scan-go-wrong-err-check-rule rule path source)] [(go-rule-id? rule "match") @@ -28433,6 +28432,35 @@ '())) '()))) +(def (go-declaration-types-before-binding binding source) + ;; Go declares `name Type` (params, struct fields, var decls), the reverse of + ;; the C/Java order. Capture the type that follows the binding identifier; + ;; keep only pointer/qualified/slice types (contain * . or [) to avoid + ;; matching `name word` prose in other languages. + (let ([name (simple-binding-identifier binding)]) + (if name + (let* ([limit (min (metavariable-binding-start-byte binding) + (string-length source))] + [prefix (substring source 0 limit)] + [pattern (string-append + "(^|[^A-Za-z0-9_$])" + (regex-escape-string name) + "[ \\t]+" + "(\\*?(?:\\[\\])?[A-Za-z_][A-Za-z0-9_.]*)")]) + (regex-fold-matches + pattern + prefix + (lambda (match acc) + (let ([type-name (re-match-group match 2)]) + (if (or (member type-name acc) + (not (or (string-find-substring type-name "*") + (string-find-substring type-name ".") + (string-find-substring type-name "[")))) + acc + (cons type-name acc)))) + '())) + '()))) + (def (inferred-binding-types binding) (let* ([text (string-trim (metavariable-binding-text binding))] [len (string-length text)]) @@ -28479,6 +28507,9 @@ (simple-declaration-types-before-binding binding source) + (go-declaration-types-before-binding + binding + source) (c-array-declaration-types-before-binding binding source))]) @@ -28862,6 +28893,11 @@ (def python-simple-assignment-regex "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*=[ \\t]*([^\\n#]+)") +(def go-simple-assignment-regex + ;; Go `name := expr` (and `name = expr`); value's first char not `=` + ;; so `==` comparisons are excluded. + "(^|\\n)[ \\t]*([A-Za-z_][A-Za-z0-9_]*)[ \\t]*(?::=|=)[ \\t]*([^\\n#=][^\\n#]*)") + (def (constant-binding-from-value name value source offset) (make-regex-capture-binding name (comparison-value->string value) @@ -28869,9 +28905,11 @@ offset offset)) -(def (constant-bindings-before source before-offset) +(def (constant-bindings-before source before-offset language) (regex-fold-matches - python-simple-assignment-regex + (if (and language (go-language? language)) + go-simple-assignment-regex + python-simple-assignment-regex) (substring source 0 before-offset) (lambda (match acc) (let* ([name (re-match-group match 2)] @@ -28915,10 +28953,11 @@ (metavariable-binding-end-line binding) (metavariable-binding-end-col binding))))) -(def (comparison-bindings-with-constants candidate source) +(def (comparison-bindings-with-constants candidate source language) (let* ([constants (constant-bindings-before source - (finding-start-offset candidate))] + (finding-start-offset candidate) + language)] [metavars (finding-metavars candidate)] [resolved (map (lambda (entry) @@ -29363,12 +29402,12 @@ (def (finding-has-resolved-decomposition? finding) (assoc 'resolved-decomposition-vars (finding-extra finding))) -(def (metavariable-comparison-satisfied? candidate clause source) +(def (metavariable-comparison-satisfied? candidate clause source language) (let* ([metavariable (alist-ref/default clause 'metavariable #f)] [comparison (alist-ref/default clause 'comparison #f)] [strip? (alist-ref/default clause 'strip #f)] [base (alist-ref/default clause 'base #f)] - [bindings (comparison-bindings-with-constants candidate source)]) + [bindings (comparison-bindings-with-constants candidate source language)]) (and comparison (not (finding-has-resolved-decomposition? candidate)) (or (not metavariable) @@ -29783,7 +29822,7 @@ (and (metavariable-analysis-satisfied? candidate (cdr clause)) candidate)] [(metavariable-comparison) - (and (metavariable-comparison-satisfied? candidate (cdr clause) source) + (and (metavariable-comparison-satisfied? candidate (cdr clause) source language) candidate)] [(focus-metavariable) candidate]