Extract Semgrep fallback positive dispatch
ober
9d18abd2b95e681d50f158cfc7c1a721f653db4d
--- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -523,6 +523,11 @@ Completed in the repo: ordering used by both plain and with-bindings scans into `src/semgrep/engine/jsts-positive-dispatch.ss`, reducing another large duplicated dispatch block in `scan.ss` + - extracted the remaining shared positive-pattern fallback ordering for + Python/symbolic/XML/YAML/C/structural scans into + `src/semgrep/engine/fallback-positive-dispatch.ss`, so the plain and + with-bindings positive scan paths now share both JS/TS and non-JS/TS + callback sequencing - exported shared Python line-assignment analysis from `src/semgrep/engine/py-constant-prop.ss` so remaining Python fallbacks in `scan.ss` can reuse one assignment-info implementation @@ -545,8 +550,8 @@ Remaining Phase 0 work: constant-propagation, f-string/interpolated, return/list, and import special-case fallback surfaces are now out, and the duplicated positive-entry dispatch core is out too; the next meaningful cuts are the remaining larger - language-specific callback ownership in `scan.ss` outside the JS/TS block, - shared textual helper ownership, and taint sections. + language-specific scanner ownership in `scan.ss`, shared textual helper + ownership, and taint sections. - Run broader upstream sweeps and grow the expected-fail baselines from real sampled data instead of one narrow seed run. new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/fallback-positive-dispatch.sls @@ -0,0 +1,49 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/fallback-positive-dispatch.ss + +(library (semgrep engine fallback-positive-dispatch) + (export + symbolic-python-like-language? + scan-fallback-positive-dispatch) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?)) + (def (symbolic-python-like-language? language) + (or (string=? language "python") + (string=? language "python2") + (string=? language "python3") + (string=? language "py"))) + (def (scan-fallback-positive-dispatch language string-key-object python-pattern-fallbacks + python-ellipsis-only python-bare-metavariable + python-method-sequence python-fstring-call + python-loop-else-constant python-and-or-string-call + python-django-redirect python-import-local + symbolic-propagation xml-pattern yaml-duplicate-id-range + c-negated-cast structural-pattern) + (or (string-key-object) + (and (symbolic-python-like-language? language) + (python-pattern-fallbacks)) + (and (symbolic-python-like-language? language) + (python-ellipsis-only)) + (and (symbolic-python-like-language? language) + (python-bare-metavariable)) + (and (symbolic-python-like-language? language) + (python-method-sequence)) + (and (symbolic-python-like-language? language) + (python-fstring-call)) + (and (symbolic-python-like-language? language) + (python-loop-else-constant)) + (and (symbolic-python-like-language? language) + (python-and-or-string-call)) + (and (symbolic-python-like-language? language) + (python-django-redirect)) + (python-import-local) + (symbolic-propagation) + (xml-pattern) + (yaml-duplicate-id-range) + (c-negated-cast) + (structural-pattern)))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -20,6 +20,9 @@ (only (semgrep engine comparison) comparison-missing? comparison-value->string constant-bindings-before comparison-value metavariable-comparison-satisfied?) + (only + (semgrep engine fallback-positive-dispatch) + scan-fallback-positive-dispatch) (semgrep engine generic-scan) (semgrep engine js-constructor-scan) (semgrep engine js-eval-scan) @@ -14896,66 +14899,78 @@ path source pattern))) - (scan-string-key-object-pattern rule path source pattern) - (and (symbolic-python-like-language? language) - (scan-python-pattern-fallbacks rule path source - pattern '())) - (and (symbolic-python-like-language? language) - (scan-python-ellipsis-only-pattern rule path source - target-root pattern)) - (and (symbolic-python-like-language? language) - (scan-python-bare-metavariable-pattern rule path - source target-root pattern)) - (and (symbolic-python-like-language? language) - (scan-python-method-sequence-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-fstring-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-loop-else-constant-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-and-or-string-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-django-redirect-pattern - rule - path - source - pattern)) - (scan-python-import-local-pattern rule language path source target-root pattern - scan-structural-pattern) - (scan-symbolic-propagation-pattern rule language path - source pattern) - (and (xml-language? language) - (scan-xml-pattern rule path source pattern)) - (and (yaml-language? language) - (scan-yaml-duplicate-id-range-pattern - rule - path - source - pattern)) - (and (c-language? language) - (scan-c-negated-cast-pattern - rule - path - source - pattern)) - (scan-structural-pattern rule language path source - target-root pattern))) + (scan-fallback-positive-dispatch language + (lambda () + (scan-string-key-object-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-pattern-fallbacks rule path source pattern + '())) + (lambda () + (scan-python-ellipsis-only-pattern rule path source + target-root pattern)) + (lambda () + (scan-python-bare-metavariable-pattern rule path source + target-root pattern)) + (lambda () + (scan-python-method-sequence-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-fstring-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-loop-else-constant-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-and-or-string-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-django-redirect-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-import-local-pattern rule language path source target-root pattern + scan-structural-pattern)) + (lambda () + (scan-symbolic-propagation-pattern rule language path + source pattern)) + (lambda () + (and (xml-language? language) + (scan-xml-pattern rule path source pattern))) + (lambda () + (and (yaml-language? language) + (scan-yaml-duplicate-id-range-pattern + rule + path + source + pattern))) + (lambda () + (and (c-language? language) + (scan-c-negated-cast-pattern + rule + path + source + pattern))) + (lambda () + (scan-structural-pattern rule language path source + target-root pattern))))) (def (scan-patterns clauses) (scan-patterns-clauses* rule clauses language path source target-root regex-captures?)) @@ -15091,57 +15106,64 @@ path source pattern))) - (and (symbolic-python-like-language? language) - (scan-python-pattern-fallbacks rule path source - pattern initial-bindings)) - (and (symbolic-python-like-language? language) - (scan-python-ellipsis-only-pattern rule path source - target-root pattern)) - (and (symbolic-python-like-language? language) - (scan-python-bare-metavariable-pattern-with-bindings rule path source target-root pattern - initial-bindings)) - (and (symbolic-python-like-language? language) - (scan-python-method-sequence-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-fstring-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-loop-else-constant-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-and-or-string-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-django-redirect-pattern - rule - path - source - pattern)) - (scan-python-import-local-pattern-with-bindings rule language path source target-root pattern - initial-bindings scan-structural-pattern-with-bindings) - (scan-symbolic-propagation-pattern rule language path - source pattern) - (and (c-language? language) - (scan-c-negated-cast-pattern - rule - path - source - pattern)) - (scan-structural-pattern-with-bindings rule language path source target-root pattern - initial-bindings))) + (scan-fallback-positive-dispatch language (lambda () #f) + (lambda () + (scan-python-pattern-fallbacks rule path source pattern + initial-bindings)) + (lambda () + (scan-python-ellipsis-only-pattern rule path source + target-root pattern)) + (lambda () + (scan-python-bare-metavariable-pattern-with-bindings rule path source target-root pattern + initial-bindings)) + (lambda () + (scan-python-method-sequence-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-fstring-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-loop-else-constant-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-and-or-string-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-django-redirect-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-import-local-pattern-with-bindings rule language path source target-root pattern + initial-bindings + scan-structural-pattern-with-bindings)) + (lambda () + (scan-symbolic-propagation-pattern rule language path + source pattern)) + (lambda () #f) (lambda () #f) + (lambda () + (and (c-language? language) + (scan-c-negated-cast-pattern + rule + path + source + pattern))) + (lambda () + (scan-structural-pattern-with-bindings rule language path source target-root pattern + initial-bindings))))) (def (scan-patterns clauses) (scan-patterns-clauses-with-bindings* rule clauses language path source target-root regex-captures? initial-bindings)) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -8,6 +8,9 @@ ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") ("src/semgrep/engine/markup-scan.ss" . "40AF9B537485FE0") ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") + ("src/semgrep/engine/fallback-positive-dispatch.ss" + . + "E6D33DDAE59E5B39") ("src/semgrep/engine/py-string-eval.ss" . "390E770972B5028B") @@ -15,16 +18,16 @@ ("src/semgrep/targeting/path-filter.ss" . "9900721941C6B96") ("src/semgrep/engine/js-eval-scan.ss" . "916D40FD4680B9BD") ("src/semgrep/result/extras.ss" . "DF0B3AAE2BAEB5D") - ("src/semgrep/engine/js-decorator-scan.ss" - . - "193758AD2E444FD6") ("src/semgrep/engine/jsts-positive-dispatch.ss" . "DABB3C66BB249070") - ("src/semgrep/engine/ts-type-scan.ss" . "B36BAF07D19F4419") - ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") - ("src/semgrep/rule.ss" . "E12C108153C181FA") + ("src/semgrep/engine/js-decorator-scan.ss" + . + "193758AD2E444FD6") ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") + ("src/semgrep/rule.ss" . "E12C108153C181FA") + ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") + ("src/semgrep/engine/ts-type-scan.ss" . "B36BAF07D19F4419") ("src/semgrep/source/offsets.ss" . "834EFDB706823794") ("src/semgrep/engine/regex-support.ss" . "9FCF118903259C97") ("src/semgrep/engine/py-constant-scan.ss" @@ -51,20 +54,20 @@ ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "2944369B60608DF1") + ("src/semgrep/scan.ss" . "FA37304B7A269959") ("src/semgrep/engine/generic-scan.ss" . "F69D0ACD0DD62610") - ("src/semgrep/engine/js-constructor-scan.ss" - . - "6B226B8D0584A7A") ("src/semgrep/engine/py-constant-prop.ss" . "76462D0EB71F2180") + ("src/semgrep/engine/js-constructor-scan.ss" + . + "6B226B8D0584A7A") ("src/semgrep/engine/comparison.ss" . "5B5731915BCB8E6") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/engine/py-import-scan.ss" . "F1817B0AFD8D9500") - ("src/semgrep/engine/text-support.ss" . "644AF29394C53045") ("src/semgrep/engine/ts-decorator-scan.ss" . - "610AA3A90D6A95F8")) + "610AA3A90D6A95F8") + ("src/semgrep/engine/text-support.ss" . "644AF29394C53045")) new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/fallback-positive-dispatch.ss @@ -0,0 +1,52 @@ +(export + symbolic-python-like-language? + scan-fallback-positive-dispatch) + +(import (except (jerboa prelude) meta atom?)) + +(def (symbolic-python-like-language? language) + (or (string=? language "python") + (string=? language "python2") + (string=? language "python3") + (string=? language "py"))) + +(def (scan-fallback-positive-dispatch + language + string-key-object + python-pattern-fallbacks + python-ellipsis-only + python-bare-metavariable + python-method-sequence + python-fstring-call + python-loop-else-constant + python-and-or-string-call + python-django-redirect + python-import-local + symbolic-propagation + xml-pattern + yaml-duplicate-id-range + c-negated-cast + structural-pattern) + (or (string-key-object) + (and (symbolic-python-like-language? language) + (python-pattern-fallbacks)) + (and (symbolic-python-like-language? language) + (python-ellipsis-only)) + (and (symbolic-python-like-language? language) + (python-bare-metavariable)) + (and (symbolic-python-like-language? language) + (python-method-sequence)) + (and (symbolic-python-like-language? language) + (python-fstring-call)) + (and (symbolic-python-like-language? language) + (python-loop-else-constant)) + (and (symbolic-python-like-language? language) + (python-and-or-string-call)) + (and (symbolic-python-like-language? language) + (python-django-redirect)) + (python-import-local) + (symbolic-propagation) + (xml-pattern) + (yaml-duplicate-id-range) + (c-negated-cast) + (structural-pattern))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -19,6 +19,8 @@ constant-bindings-before comparison-value metavariable-comparison-satisfied?) + (only (semgrep engine fallback-positive-dispatch) + scan-fallback-positive-dispatch) (semgrep engine generic-scan) (semgrep engine js-constructor-scan) (semgrep engine js-eval-scan) @@ -14802,101 +14804,110 @@ path source pattern))) - (scan-string-key-object-pattern - rule - path - source - pattern) - (and (symbolic-python-like-language? language) - (scan-python-pattern-fallbacks - rule - path - source - pattern - '())) - (and (symbolic-python-like-language? language) - (scan-python-ellipsis-only-pattern - rule - path - source - target-root - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-bare-metavariable-pattern - rule - path - source - target-root - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-method-sequence-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-fstring-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-loop-else-constant-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-and-or-string-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-django-redirect-pattern - rule - path - source - pattern)) - (scan-python-import-local-pattern - rule + (scan-fallback-positive-dispatch language - path - source - target-root - pattern - scan-structural-pattern) - (scan-symbolic-propagation-pattern - rule - language - path - source - pattern) - (and (xml-language? language) - (scan-xml-pattern - rule - path - source - pattern)) - (and (yaml-language? language) - (scan-yaml-duplicate-id-range-pattern - rule - path - source - pattern)) - (and (c-language? language) - (scan-c-negated-cast-pattern - rule - path - source - pattern)) - (scan-structural-pattern - rule - language - path - source - target-root - pattern))) + (lambda () + (scan-string-key-object-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-pattern-fallbacks + rule + path + source + pattern + '())) + (lambda () + (scan-python-ellipsis-only-pattern + rule + path + source + target-root + pattern)) + (lambda () + (scan-python-bare-metavariable-pattern + rule + path + source + target-root + pattern)) + (lambda () + (scan-python-method-sequence-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-fstring-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-loop-else-constant-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-and-or-string-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-django-redirect-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-import-local-pattern + rule + language + path + source + target-root + pattern + scan-structural-pattern)) + (lambda () + (scan-symbolic-propagation-pattern + rule + language + path + source + pattern)) + (lambda () + (and (xml-language? language) + (scan-xml-pattern + rule + path + source + pattern))) + (lambda () + (and (yaml-language? language) + (scan-yaml-duplicate-id-range-pattern + rule + path + source + pattern))) + (lambda () + (and (c-language? language) + (scan-c-negated-cast-pattern + rule + path + source + pattern))) + (lambda () + (scan-structural-pattern + rule + language + path + source + target-root + pattern))))) (def (scan-patterns clauses) (scan-patterns-clauses* rule @@ -15111,87 +15122,96 @@ path source pattern))) - (and (symbolic-python-like-language? language) - (scan-python-pattern-fallbacks - rule - path - source - pattern - initial-bindings)) - (and (symbolic-python-like-language? language) - (scan-python-ellipsis-only-pattern - rule - path - source - target-root - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-bare-metavariable-pattern-with-bindings - rule - path - source - target-root - pattern - initial-bindings)) - (and (symbolic-python-like-language? language) - (scan-python-method-sequence-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-fstring-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-loop-else-constant-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-and-or-string-call-pattern - rule - path - source - pattern)) - (and (symbolic-python-like-language? language) - (scan-python-django-redirect-pattern - rule - path - source - pattern)) - (scan-python-import-local-pattern-with-bindings - rule + (scan-fallback-positive-dispatch language - path - source - target-root - pattern - initial-bindings - scan-structural-pattern-with-bindings) - (scan-symbolic-propagation-pattern - rule - language - path - source - pattern) - (and (c-language? language) - (scan-c-negated-cast-pattern - rule - path - source - pattern)) - (scan-structural-pattern-with-bindings - rule - language - path - source - target-root - pattern - initial-bindings))) + (lambda () #f) + (lambda () + (scan-python-pattern-fallbacks + rule + path + source + pattern + initial-bindings)) + (lambda () + (scan-python-ellipsis-only-pattern + rule + path + source + target-root + pattern)) + (lambda () + (scan-python-bare-metavariable-pattern-with-bindings + rule + path + source + target-root + pattern + initial-bindings)) + (lambda () + (scan-python-method-sequence-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-fstring-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-loop-else-constant-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-and-or-string-call-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-django-redirect-pattern + rule + path + source + pattern)) + (lambda () + (scan-python-import-local-pattern-with-bindings + rule + language + path + source + target-root + pattern + initial-bindings + scan-structural-pattern-with-bindings)) + (lambda () + (scan-symbolic-propagation-pattern + rule + language + path + source + pattern)) + (lambda () #f) + (lambda () #f) + (lambda () + (and (c-language? language) + (scan-c-negated-cast-pattern + rule + path + source + pattern))) + (lambda () + (scan-structural-pattern-with-bindings + rule + language + path + source + target-root + pattern + initial-bindings))))) (def (scan-patterns clauses) (scan-patterns-clauses-with-bindings* rule