Replace go/int-binop overfit handler with expression-int inference
ober
567a2534bcb2682dbc6b8c9fd986285769b6496b
--- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -26696,8 +26696,6 @@ [(go-rule-id? rule "asymmetric-rsa-weak-keylength") '()] [(go-rule-id? rule "use-of-weak-rsa-key") (scan-go-rsa-weak-key-rule rule path source)] - [(go-rule-id? rule "int-binop") - (scan-go-int-binop-rule rule path source)] [(go-rule-id? rule "reinstantiated_variable_in_new_block") (scan-go-reinstantiated-var-rule rule path source)] [(go-rule-id? rule "typed-metavar-metavar-regex") @@ -28546,6 +28544,91 @@ (cons type-name acc)))) '())) '()))) + (def (arith-operator-char? ch) + (or (char=? ch #\+) + (char=? ch #\-) + (char=? ch #\*) + (char=? ch #\/) + (char=? ch #\%))) + (def (split-top-arithmetic expr) + (let ([len (string-length expr)]) + (let loop ([i 1] [pd 0]) + (cond + [(>= i len) #f] + [(char=? (string-ref expr i) #\() (loop (+ i 1) (+ pd 1))] + [(char=? (string-ref expr i) #\)) + (loop (+ i 1) (max 0 (- pd 1)))] + [(and (= pd 0) + (arith-operator-char? (string-ref expr i)) + (not (arith-operator-char? (string-ref expr (- i 1))))) + (cons (substring expr 0 i) (substring expr (+ i 1) len))] + [else (loop (+ i 1) pd)])))) + (def (simple-identifier-expr? s) + (and (> (string-length s) 0) + (let ([c0 (string-ref s 0)]) + (or (char-alphabetic? c0) (char=? c0 #\_))) + (let loop ([i 1]) + (or (>= i (string-length s)) + (and (identifier-token-char? (string-ref s i)) + (loop (+ i 1))))))) + (def (const-value-before name source before) + (let ([pattern (string-append + "(^|[^A-Za-z0-9_])" + (regex-escape-string name) + "[ \\t]*(?::=|=)[ \\t]*([^\\n;]+)")]) + (regex-fold-matches + pattern + (substring source 0 (min before (string-length source))) + (lambda (match acc) + (or acc (string-trim (re-match-group match 2)))) + #f))) + (def (expression-int? expr source before depth) + (and (< depth 8) + (let ([t (string-trim expr)]) + (and (> (string-length t) 0) + (cond + [(and (char=? (string-ref t 0) #\() + (char=? + (string-ref t (- (string-length t) 1)) + #\))) + (expression-int? + (substring t 1 (- (string-length t) 1)) + source + before + (+ depth 1))] + [(split-top-arithmetic t) => + (lambda (p) + (and (expression-int? + (car p) + source + before + (+ depth 1)) + (expression-int? + (cdr p) + source + before + (+ depth 1))))] + [(parse-number-literal t #f) + (not (numeric-literal-float-like? t))] + [(quoted-string? t) #f] + [(simple-identifier-expr? t) + (let ([v (const-value-before t source before)]) + (and v + (not (string=? v t)) + (expression-int? + v + source + before + (+ depth 1))))] + [else #f]))))) + (def (expression-int-types-before-binding binding source) + (if (expression-int? + (metavariable-binding-text binding) + source + (metavariable-binding-start-byte binding) + 0) + '("int" "integer" "number") + '())) (def (inferred-binding-types binding) (let* ([text (string-trim (metavariable-binding-text binding))] @@ -28592,6 +28675,9 @@ metavariable))]) (and binding (let ([actual-types (append (inferred-binding-types binding) + (expression-int-types-before-binding + binding + source) (annotation-types-before-binding binding source) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -3,11 +3,11 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "B1616180DE7038ED") - ("src/semgrep/scan.ss" . "60922ED6A9C2992B") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/scan.ss" . "75B176B4743AB7E1") ("src/semgrep/rule.ss" . "E12C108153C181FA") - ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") ("src/semgrep/match/structural.ss" . "AA1BA746924C8189") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -26675,8 +26675,8 @@ [(go-rule-id? rule "asymmetric-rsa-weak-keylength") '()] [(go-rule-id? rule "use-of-weak-rsa-key") (scan-go-rsa-weak-key-rule rule path source)] - [(go-rule-id? rule "int-binop") - (scan-go-int-binop-rule rule path source)] + ;; int-binop now handled by structural matching + metavariable-type:int + ;; with expression-int inference (expression-int-types-before-binding). [(go-rule-id? rule "reinstantiated_variable_in_new_block") (scan-go-reinstantiated-var-rule rule path source)] ;; metavar-type-func-param + no-direct-db-exec are now handled by the @@ -28461,6 +28461,75 @@ '())) '()))) +(def (arith-operator-char? ch) + (or (char=? ch #\+) (char=? ch #\-) (char=? ch #\*) + (char=? ch #\/) (char=? ch #\%))) + +(def (split-top-arithmetic expr) + ;; Split on the first top-level binary arithmetic operator (skips a leading + ;; unary sign and operators that follow another operator). (left . right)|#f. + (let ([len (string-length expr)]) + (let loop ([i 1] [pd 0]) + (cond + [(>= i len) #f] + [(char=? (string-ref expr i) #\() (loop (+ i 1) (+ pd 1))] + [(char=? (string-ref expr i) #\)) (loop (+ i 1) (max 0 (- pd 1)))] + [(and (= pd 0) + (arith-operator-char? (string-ref expr i)) + (not (arith-operator-char? (string-ref expr (- i 1))))) + (cons (substring expr 0 i) (substring expr (+ i 1) len))] + [else (loop (+ i 1) pd)])))) + +(def (simple-identifier-expr? s) + (and (> (string-length s) 0) + (let ([c0 (string-ref s 0)]) + (or (char-alphabetic? c0) (char=? c0 #\_))) + (let loop ([i 1]) + (or (>= i (string-length s)) + (and (identifier-token-char? (string-ref s i)) (loop (+ i 1))))))) + +(def (const-value-before name source before) + ;; First `name = expr` / `name := expr` (e.g. a const/var) before `before`. + (let ([pattern (string-append + "(^|[^A-Za-z0-9_])" + (regex-escape-string name) + "[ \\t]*(?::=|=)[ \\t]*([^\\n;]+)")]) + (regex-fold-matches + pattern + (substring source 0 (min before (string-length source))) + (lambda (match acc) (or acc (string-trim (re-match-group match 2)))) + #f))) + +(def (expression-int? expr source before depth) + (and (< depth 8) + (let ([t (string-trim expr)]) + (and (> (string-length t) 0) + (cond + [(and (char=? (string-ref t 0) #\() + (char=? (string-ref t (- (string-length t) 1)) #\))) + (expression-int? (substring t 1 (- (string-length t) 1)) + source before (+ depth 1))] + [(split-top-arithmetic t) + => (lambda (p) + (and (expression-int? (car p) source before (+ depth 1)) + (expression-int? (cdr p) source before (+ depth 1))))] + [(parse-number-literal t #f) + (not (numeric-literal-float-like? t))] + [(quoted-string? t) #f] + [(simple-identifier-expr? t) + (let ([v (const-value-before t source before)]) + (and v (not (string=? v t)) + (expression-int? v source before (+ depth 1))))] + [else #f]))))) + +(def (expression-int-types-before-binding binding source) + (if (expression-int? (metavariable-binding-text binding) + source + (metavariable-binding-start-byte binding) + 0) + '("int" "integer" "number") + '())) + (def (inferred-binding-types binding) (let* ([text (string-trim (metavariable-binding-text binding))] [len (string-length text)]) @@ -28501,6 +28570,9 @@ (finding-metavariable-binding candidate metavariable))]) (and binding (let ([actual-types (append (inferred-binding-types binding) + (expression-int-types-before-binding + binding + source) (annotation-types-before-binding binding source)