Add upstream oracle baselines and split scan planning
ober
0d692d2aa131ee8e2b32961b427e911b69f50444
--- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ LIBDIRS = lib:$(JERBOA_TREESITTER_HOME)/lib:$(JERBOA_HOME)/lib PATTERN_CASE_REGEX ?= ^(ac_matching_dots|ac_matching_dots1|ac_matching_explosion|ac_matching_explosion1|ac_matching_explosion2|ac_matching_explosion3|ac_matching_free|ac_matching_free1|ac_matching_if|ac_matching_if1|ac_matching_mvars|ac_matching_mvars1|ac_matching_mvars2|aliasing_attribute|assoc_matching_bug|assoc_matching_dots|assoc_matching_dots1|assoc_matching_explosion|assoc_matching_explosion1|assoc_matching_explosion2|assoc_matching_explosion3|assoc_matching_free|assoc_matching_free1|assoc_matching_mvars|assoc_matching_mvars1|constprop_dataflow|constprop_used_twice|cp_concat|cp_exception|cp_label|cp_label1|cp_method_call|cp_python_mult_string|cp_python_mult_string1|cp_rlval|cp_string_mvar|cp_strings|cp_with|cp_yield|deep_cond|deep_expr_operator|deep_expr_vs_statement|deep_exprstmt|deep_fake_dataflow|deep_try|dict_ellipsis|dots_attribute|dots_expr_plus|dots_expr_plus_string|dots_fstring|dots_fstring_with_match_stmt|dots_inherit|dots_list|dots_nested_stmts|dots_stmts|equivalence_constant_propagation|equivalence_f_string|equivalence_f_string_2|equivalence_f_string_3|equivalence_f_string_4|equivalence_interpolated_str|equivalence_interpolated_str2|equivalence_keyword_args|equivalence_naming_import|import_metavar|import_negatives|import_negatives2|imports|index_tuple|less_attributes|less_catch|less_catch_multiple|less_inherits|less_typehint|less_typehint2|matching_if_expr|metavar_anno|metavar_anno_fqn|metavar_class_def|metavar_dict|metavar_equality_param_vs_use|metavar_equality_var|metavar_func_def|metavar_stmt|metavar_typed|misc_attributes|misc_block_import|misc_class_regression|misc_dots_stmts|misc_except_matching|misc_faketok2|misc_if1|misc_metavar_vs_fstring|misc_naming_bug2|misc_paren|misc_range_statements|misc_regression1|misc_regression2|misc_stmts1|misc_tuple2|misc_with_parens|multi_import|multi_qualified_wildcard|pip614-extended-decorator-grammer1|regexp_string_backref|set_vs_dict|set_vs_dict2|standalone_decorator|standalone_decorator_single_arg|static_method|stmts_to_fields|tuple_subscript_ellipsis|tuple_subscript_ellipsis_with_match_stmt|unordered_metavar2|untyped_vs_typed|wildcard_qualified)$$ JS_PATTERN_CASE_REGEX ?= ^(ac_matching_bitor|aliasing_require|caching_deep|caching_deep_metavar|caching_different_depth|caching_flat|caching_nested|cp_conditional|cp_dataflow|cp_implicit_conversion|cp_incrdecr|cp_label|cp_throw|cp_undeclared|cp_undeclared1|deep_cond|deep_deep_stmt1|deep_deep_stmt2|deep_expr_and_ellipsis|deep_expr_vs_statement|deep_expr_xml|deep_stmt|deep_stmt2|deep_vardef|dots_field_chaining|dots_importfrom|dots_jsx_attr|dots_jsx_body|dots_jsx_body2|dots_template_literals|ellipsis_metavar_tagged_template|ellipsis_metavar_template|equivalence_aliasing_import|equivalence_import_require|equivalence_import_variations|equivalence_import_variations2|equivalence_import_variations3|equivalence_import_variations4|equivalence_keyword_args|equivalence_varlet|infer_const|infer_const_1|infer_const_regexp|equivalence_constant_propagation2|equivalence_constant_propagation3|less_class_complex|less_class_complex_2|less_inheritance|less_xml_attr|less_xml_body|metavar_class_fields|metavar_equality_vardef_vs_use|metavar_import|metavar_import2|metavar_import3|metavar_importfrom|metavar_importfrom2|metavar_jsx_val|metavar_stmt_or_expr|metavar_template|metavar_typed_bool|metavar_typed_literal|metavar_xml_attr|misc_arrow|misc_arrow2|misc_asi_pattern|misc_catch_unbound|misc_decorators_method|misc_deep_expr_stmt|misc_empty_body|misc_export|misc_method_chain_vs_stmt|misc_object_directly|misc_tagged_template_literal|object_duplicate_key|object_numeric_key|partial_catch|partial_decorators_arg_method|partial_decorators_method|partial_function|partial_lambda_or_func|partial_single_field|regexp_fieldname|switch_case_pattern|template_string_w_metavar|typed_metavar_assign|unordered_destructuring|unordered_fields|unordered_fields2|unordered_metavar|unordered_xml_attr)$$ -.PHONY: all build generate test oracle patterns-oracle patterns-js-oracle clean +.PHONY: all build generate test oracle oracle-upstream patterns-oracle patterns-js-oracle clean all: build @@ -33,6 +33,13 @@ oracle: build LIBDIRS="$(LIBDIRS)" \ tests/oracle/run.sh +oracle-upstream: build + JERBOA_HOME="$(JERBOA_HOME)" \ + JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ + SCHEME="$(SCHEME)" \ + LIBDIRS="$(LIBDIRS)" \ + tests/oracle/upstream-sweep.sh + patterns-oracle: build JERBOA_HOME="$(JERBOA_HOME)" \ JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ --- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -429,6 +429,48 @@ The harness should support: `/Users/user/mine/semgrep`. - Stable JSON output normalization before comparison. +## Progress Log + +### 2026-06-03 Phase 0 checkpoint + +Completed in the repo: + +- Added `make oracle-upstream` as a first-class entry point for upstream rule + sweeps. +- Added reusable expected-fail baseline support in + `tests/oracle/expected-fails.sh`. +- Updated `tests/oracle/upstream-sweep.sh` and + `tests/oracle/patterns-sweep.sh` to support tracked expected failures and + baseline regeneration. +- Added tracked baseline files: + - `tests/oracle/upstream-expected-fails.txt` + - `tests/oracle/patterns-expected-fails.txt` +- Recorded the first upstream-sweep baseline entries for upstream/current + Semgrep `current-error` cases: + - `anywhere_global` + - `anywhere_include` + - `anywhere_metavar` +- Started the `src/semgrep/scan.ss` split without changing public `scan-*` + entry points: + - extracted path/file filtering into `src/semgrep/targeting/path-filter.ss` + - extracted rule applicability and parse gating into + `src/semgrep/engine/rule-plan.ss` + +Validation at this checkpoint: + +- `make test`: 321/321 passing +- `make oracle`: 42/42 passing +- Sample upstream rule sweep: + `MAX_CASES=20 MAX_DIFFS=2 tests/oracle/upstream-sweep.sh` + passes with the tracked expected-fail baseline + +Remaining Phase 0 work: + +- Continue splitting `src/semgrep/scan.ss` into engine, targeting, text-mode, + structural dispatch, taint, and result/output-adjacent modules. +- Run broader upstream sweeps and grow the expected-fail baselines from real + sampled data instead of one narrow seed run. + ## Complete Implementation Phases ### Phase 0: Stabilize the Current Port new file mode 100644 --- /dev/null +++ b/lib/semgrep/engine/rule-plan.sls @@ -0,0 +1,54 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/engine/rule-plan.ss + +(library (semgrep engine rule-plan) + (export + language-applies? + rule-needs-parse? + applicable-rules) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) (semgrep lang) + (semgrep rule) (semgrep targeting path-filter)) + (def (language-applies? rule language) + (let ([canonical (or (canonical-language language) + language)]) + (member canonical (rule-languages rule)))) + (def (positive-entry-needs-parse? entry) + (case (car entry) + [(pattern-regex) #f] + [(pattern) #t] + [(pattern-either) + (any? positive-entry-needs-parse? (cdr entry))] + [(patterns) (any? pattern-clause-needs-parse? (cdr entry))] + [(pattern-as) + (positive-entry-needs-parse? + (let ([nested (assoc 'entry (cdr entry))]) + (and nested (cdr nested))))] + [else #t])) + (def (pattern-clause-needs-parse? clause) + (case (car clause) + [(pattern-regex) #f] + [(pattern-not) (positive-entry-needs-parse? (cdr clause))] + [(metavariable-regex focus-metavariable) #f] + [else #t])) + (def (rule-needs-parse? rule) + (case (rule-pattern-kind rule) + [(pattern-regex) #f] + [(pattern-either) + (any? positive-entry-needs-parse? (rule-pattern rule))] + [(patterns) + (any? pattern-clause-needs-parse? (rule-pattern rule))] + [else #t])) + (def (applicable-rules rules language path) + (let loop ([remaining rules] [acc '()]) + (cond + [(null? remaining) (reverse acc)] + [(and (language-applies? (car remaining) language) + (path-applies? (car remaining) path)) + (loop (cdr remaining) (cons (car remaining) acc))] + [else (loop (cdr remaining) acc)])))) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -14,41 +14,10 @@ with-input-from-string with-output-to-string iota \x31;+ \x31;- partition make-date make-time meta atom?) (except (jerboa prelude) meta atom?) (std regex) - (std text glob) (tree-sitter tree-sitter) (semgrep lang) - (semgrep rule) (semgrep result) (semgrep rule parse-rule) - (semgrep parse parse-target) (semgrep match structural)) - (def (read-target-file-string path) - (call-with-input-file - path - (lambda (port) - (let ([source (get-string-all port)]) - (if (eof-object? source) "" source))))) - (def (language-applies? rule language) - (let ([canonical (or (canonical-language language) - language)]) - (member canonical (rule-languages rule)))) - (def (path-basename path) - (let ([len (string-length path)]) - (let loop ([i (- len 1)]) - (cond - [(< i 0) path] - [(char=? (string-ref path i) #\/) - (substring path (+ i 1) len)] - [else (loop (- i 1))])))) - (def (path-pattern-matches? pattern path) - (or (glob-match? pattern path) - (glob-match? pattern (path-basename path)))) - (def (path-applies? rule path) - (let ([includes (rule-path-includes rule)] - [excludes (rule-path-excludes rule)]) - (and (or (null? includes) - (any? - (lambda (pattern) (path-pattern-matches? pattern path)) - includes)) - (not (any? - (lambda (pattern) - (path-pattern-matches? pattern path)) - excludes))))) + (tree-sitter tree-sitter) (semgrep lang) (semgrep rule) + (semgrep result) (semgrep engine rule-plan) + (semgrep rule parse-rule) (semgrep parse parse-target) + (semgrep targeting path-filter) (semgrep match structural)) (def (alist-ref/default xs key default) (let ([found (assoc key xs)]) (if found (cdr found) default))) @@ -70,11 +39,6 @@ [(pred (car remaining)) (loop (cdr remaining) (cons (car remaining) acc))] [else (loop (cdr remaining) acc)]))) - (def (any? pred xs) - (and (not (null? xs)) - (or (pred (car xs)) (any? pred (cdr xs))))) - (def (all? pred xs) - (or (null? xs) (and (pred (car xs)) (all? pred (cdr xs))))) (def (remove-first-eq target xs) (cond [(null? xs) '()] @@ -34370,37 +34334,11 @@ (error 'scan-string "unsupported rule pattern kind" (rule-pattern-kind rule))]))) - (def (positive-entry-needs-parse? entry) - (case (car entry) - [(pattern-regex) #f] - [(pattern) #t] - [(pattern-either) - (any? positive-entry-needs-parse? (cdr entry))] - [(patterns) (any? pattern-clause-needs-parse? (cdr entry))] - [(pattern-as) - (positive-entry-needs-parse? - (alist-ref/default (cdr entry) 'entry #f))] - [else #t])) - (def (pattern-clause-needs-parse? clause) - (case (car clause) - [(pattern-regex) #f] - [(pattern-not) (positive-entry-needs-parse? (cdr clause))] - [(metavariable-regex focus-metavariable) #f] - [else #t])) - (def (rule-needs-parse? rule) - (case (rule-pattern-kind rule) - [(pattern-regex) #f] - [(pattern-either) - (any? positive-entry-needs-parse? (rule-pattern rule))] - [(patterns) - (any? pattern-clause-needs-parse? (rule-pattern rule))] - [else #t])) (def (scan-string rules language path source) - (let* ([applicable-rules (sg-filter - (lambda (rule) - (and (language-applies? rule language) - (path-applies? rule path))) - rules)] + (let* ([applicable-rules (applicable-rules + rules + language + path)] [parse-result (and (not (generic-language? language)) (any? rule-needs-parse? applicable-rules) (parse-target-string language source))] new file mode 100644 --- /dev/null +++ b/lib/semgrep/targeting/path-filter.sls @@ -0,0 +1,48 @@ +#!chezscheme +;;; Generated by jerbuild — DO NOT EDIT +;;; Source: src/semgrep/targeting/path-filter.ss + +(library (semgrep targeting path-filter) + (export read-target-file-string any? all? path-basename + path-pattern-matches? path-applies?) + (import + (except (chezscheme) make-hash-table hash-table? sort sort! + printf fprintf format path-extension path-absolute? + with-input-from-string with-output-to-string iota \x31;+ + \x31;- partition make-date make-time meta atom?) + (except (jerboa prelude) meta atom?) + (std text glob) + (semgrep rule)) + (def (read-target-file-string path) + (call-with-input-file + path + (lambda (port) + (let ([source (get-string-all port)]) + (if (eof-object? source) "" source))))) + (def (any? pred xs) + (and (not (null? xs)) + (or (pred (car xs)) (any? pred (cdr xs))))) + (def (all? pred xs) + (or (null? xs) (and (pred (car xs)) (all? pred (cdr xs))))) + (def (path-basename path) + (let ([len (string-length path)]) + (let loop ([i (- len 1)]) + (cond + [(< i 0) path] + [(char=? (string-ref path i) #\/) + (substring path (+ i 1) len)] + [else (loop (- i 1))])))) + (def (path-pattern-matches? pattern path) + (or (glob-match? pattern path) + (glob-match? pattern (path-basename path)))) + (def (path-applies? rule path) + (let ([includes (rule-path-includes rule)] + [excludes (rule-path-excludes rule)]) + (and (or (null? includes) + (any? + (lambda (pattern) (path-pattern-matches? pattern path)) + includes)) + (not (any? + (lambda (pattern) + (path-pattern-matches? pattern path)) + excludes)))))) --- a/src/.jerbuild-hashes +++ b/src/.jerbuild-hashes @@ -1,13 +1,17 @@ -(("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") ("src/semgrep/rule/parse-rule.ss" . "EFA6D401699CEDEF") +(("src/semgrep/targeting/path-filter.ss" + . + "9900721941C6B96") ("src/semgrep/output/sarif.ss" . "E935456E4B1921FB") + ("src/semgrep/rule/parse-rule.ss" . "EFA6D401699CEDEF") ("src/semgrep/result.ss" . "22D23E40B49BA529") ("src/semgrep/output/json.ss" . "293881CFA2ADB7BC") + ("src/semgrep/engine/rule-plan.ss" . "6631789392AB5F80") ("src/semgrep/lang.ss" . "6982E07679D20836") ("src/semgrep/parse/parse-target.ss" . "97AA8FFEB12736DA") - ("src/semgrep/scan.ss" . "6F8670D0036F43DD") + ("src/semgrep/scan.ss" . "E2D828980E2073F1") + ("src/semgrep/rule.ss" . "E12C108153C181FA") + ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") ("src/semgrep/output/text.ss" . "BE476CB84B807FBA") ("src/semgrep/fix.ss" . "2E5B65B1FEF3B2B1") - ("src/semgrep/schema/lang.ss" . "CAE2CA859C9A9FD0") - ("src/semgrep/rule.ss" . "E12C108153C181FA") ("src/semgrep/match/structural.ss" . "5BA4F1566448AF3A") ("src/semgrep/main.ss" . "A4EC9E7F2A09D25E") ("src/semgrep/cli.ss" . "EBDC4B1DAD3F13CC")) new file mode 100644 --- /dev/null +++ b/src/semgrep/engine/rule-plan.ss @@ -0,0 +1,53 @@ +(export + language-applies? + rule-needs-parse? + applicable-rules) + +(import (except (jerboa prelude) meta atom?) + (semgrep lang) + (semgrep rule) + (semgrep targeting path-filter)) + +(def (language-applies? rule language) + (let ([canonical (or (canonical-language language) language)]) + (member canonical (rule-languages rule)))) + +(def (positive-entry-needs-parse? entry) + (case (car entry) + [(pattern-regex) #f] + [(pattern) #t] + [(pattern-either) + (any? positive-entry-needs-parse? (cdr entry))] + [(patterns) + (any? pattern-clause-needs-parse? (cdr entry))] + [(pattern-as) + (positive-entry-needs-parse? + (let ([nested (assoc 'entry (cdr entry))]) + (and nested (cdr nested))))] + [else #t])) + +(def (pattern-clause-needs-parse? clause) + (case (car clause) + [(pattern-regex) #f] + [(pattern-not) + (positive-entry-needs-parse? (cdr clause))] + [(metavariable-regex focus-metavariable) #f] + [else #t])) + +(def (rule-needs-parse? rule) + (case (rule-pattern-kind rule) + [(pattern-regex) #f] + [(pattern-either) + (any? positive-entry-needs-parse? (rule-pattern rule))] + [(patterns) + (any? pattern-clause-needs-parse? (rule-pattern rule))] + [else #t])) + +(def (applicable-rules rules language path) + (let loop ([remaining rules] [acc '()]) + (cond + [(null? remaining) (reverse acc)] + [(and (language-applies? (car remaining) language) + (path-applies? (car remaining) path)) + (loop (cdr remaining) (cons (car remaining) acc))] + [else (loop (cdr remaining) acc)]))) --- a/src/semgrep/scan.ss +++ b/src/semgrep/scan.ss @@ -6,49 +6,16 @@ (import (except (jerboa prelude) meta atom?) (std regex) - (std text glob) (tree-sitter tree-sitter) (semgrep lang) (semgrep rule) (semgrep result) + (semgrep engine rule-plan) (semgrep rule parse-rule) (semgrep parse parse-target) + (semgrep targeting path-filter) (semgrep match structural)) -(def (read-target-file-string path) - (call-with-input-file path - (lambda (port) - (let ([source (get-string-all port)]) - (if (eof-object? source) "" source))))) - -(def (language-applies? rule language) - (let ([canonical (or (canonical-language language) language)]) - (member canonical (rule-languages rule)))) - -(def (path-basename path) - (let ([len (string-length path)]) - (let loop ([i (- len 1)]) - (cond - [(< i 0) path] - [(char=? (string-ref path i) #\/) - (substring path (+ i 1) len)] - [else (loop (- i 1))])))) - -(def (path-pattern-matches? pattern path) - (or (glob-match? pattern path) - (glob-match? pattern (path-basename path)))) - -(def (path-applies? rule path) - (let ([includes (rule-path-includes rule)] - [excludes (rule-path-excludes rule)]) - (and (or (null? includes) - (any? (lambda (pattern) - (path-pattern-matches? pattern path)) - includes)) - (not (any? (lambda (pattern) - (path-pattern-matches? pattern path)) - excludes))))) - (def (alist-ref/default xs key default) (let ([found (assoc key xs)]) (if found (cdr found) default))) @@ -77,16 +44,6 @@ (loop (cdr remaining) (cons (car remaining) acc))] [else (loop (cdr remaining) acc)]))) -(def (any? pred xs) - (and (not (null? xs)) - (or (pred (car xs)) - (any? pred (cdr xs))))) - -(def (all? pred xs) - (or (null? xs) - (and (pred (car xs)) - (all? pred (cdr xs))))) - (def (remove-first-eq target xs) (cond [(null? xs) '()] @@ -34348,45 +34305,12 @@ [else (error 'scan-string "unsupported rule pattern kind" (rule-pattern-kind rule))]))) -(def (positive-entry-needs-parse? entry) - (case (car entry) - [(pattern-regex) #f] - [(pattern) #t] - [(pattern-either) - (any? positive-entry-needs-parse? (cdr entry))] - [(patterns) - (any? pattern-clause-needs-parse? (cdr entry))] - [(pattern-as) - (positive-entry-needs-parse? - (alist-ref/default (cdr entry) 'entry #f))] - [else #t])) - -(def (pattern-clause-needs-parse? clause) - (case (car clause) - [(pattern-regex) #f] - [(pattern-not) - (positive-entry-needs-parse? (cdr clause))] - [(metavariable-regex focus-metavariable) #f] - [else #t])) - -(def (rule-needs-parse? rule) - (case (rule-pattern-kind rule) - [(pattern-regex) #f] - [(pattern-either) - (any? positive-entry-needs-parse? (rule-pattern rule))] - [(patterns) - (any? pattern-clause-needs-parse? (rule-pattern rule))] - [else #t])) - (def (scan-string rules language path source) ;; Regex-only rules can run without a parser. Structural rules still parse ;; first so unsupported languages and serious syntax issues are visible before ;; matching. (let* ([applicable-rules - (sg-filter (lambda (rule) - (and (language-applies? rule language) - (path-applies? rule path))) - rules)] + (applicable-rules rules language path)] [parse-result (and (not (generic-language? language)) (any? rule-needs-parse? applicable-rules) (parse-target-string language source))] new file mode 100644 --- /dev/null +++ b/src/semgrep/targeting/path-filter.ss @@ -0,0 +1,51 @@ +(export + read-target-file-string + any? + all? + path-basename + path-pattern-matches? + path-applies?) + +(import (except (jerboa prelude) meta atom?) + (std text glob) + (semgrep rule)) + +(def (read-target-file-string path) + (call-with-input-file path + (lambda (port) + (let ([source (get-string-all port)]) + (if (eof-object? source) "" source))))) + +(def (any? pred xs) + (and (not (null? xs)) + (or (pred (car xs)) + (any? pred (cdr xs))))) + +(def (all? pred xs) + (or (null? xs) + (and (pred (car xs)) + (all? pred (cdr xs))))) + +(def (path-basename path) + (let ([len (string-length path)]) + (let loop ([i (- len 1)]) + (cond + [(< i 0) path] + [(char=? (string-ref path i) #\/) + (substring path (+ i 1) len)] + [else (loop (- i 1))])))) + +(def (path-pattern-matches? pattern path) + (or (glob-match? pattern path) + (glob-match? pattern (path-basename path)))) + +(def (path-applies? rule path) + (let ([includes (rule-path-includes rule)] + [excludes (rule-path-excludes rule)]) + (and (or (null? includes) + (any? (lambda (pattern) + (path-pattern-matches? pattern path)) + includes)) + (not (any? (lambda (pattern) + (path-pattern-matches? pattern path)) + excludes))))) new file mode 100644 --- /dev/null +++ b/tests/oracle/expected-fails.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +oracle_expectations_init() { + local tmpdir="$1" + local expected_file="${2:-}" + local baseline_out="${3:-}" + + ORACLE_EXPECTED_IDS="$tmpdir/expected.ids" + ORACLE_BASELINE_TMP="$tmpdir/baseline.ids" + ORACLE_BASELINE_OUT="$baseline_out" + ORACLE_EXPECTED_FAILURES=0 + ORACLE_UNEXPECTED_FAILURES=0 + ORACLE_UNEXPECTED_PASSES=0 + + : >"$ORACLE_EXPECTED_IDS" + : >"$ORACLE_BASELINE_TMP" + + if [[ -n "$expected_file" && -f "$expected_file" ]]; then + awk ' + NF == 0 { next } + $1 ~ /^#/ { next } + { print $1 } + ' "$expected_file" | sort -u >"$ORACLE_EXPECTED_IDS" + fi +} + +oracle_expected_case() { + local case_id="$1" + grep -Fxq "$case_id" "$ORACLE_EXPECTED_IDS" +} + +oracle_record_failure() { + local case_id="$1" + local outcome="$2" + + if [[ -n "$ORACLE_BASELINE_OUT" ]]; then + printf '%s %s\n' "$case_id" "$outcome" >>"$ORACLE_BASELINE_TMP" + fi + + if oracle_expected_case "$case_id"; then + ORACLE_EXPECTED_FAILURES=$((ORACLE_EXPECTED_FAILURES + 1)) + if [[ "${LIST_MISMATCHES:-0}" != "0" ]]; then + echo "EXPECTED-FAIL $outcome $case_id" + fi + return 0 + fi + + ORACLE_UNEXPECTED_FAILURES=$((ORACLE_UNEXPECTED_FAILURES + 1)) + return 1 +} + +oracle_record_pass() { + local case_id="$1" + + if oracle_expected_case "$case_id"; then + ORACLE_UNEXPECTED_PASSES=$((ORACLE_UNEXPECTED_PASSES + 1)) + if [[ "${LIST_MISMATCHES:-0}" != "0" ]]; then + echo "UNEXPECTED-PASS $case_id" + fi + return 1 + fi + + return 0 +} + +oracle_write_baseline() { + local label="$1" + local command_hint="$2" + + if [[ -z "$ORACLE_BASELINE_OUT" ]]; then + return 0 + fi + + { + printf '# Expected-fail baseline for %s\n' "$label" + printf '# Format: <case-id> <last-observed-outcome>\n' + printf '# Regenerate with: %s\n' "$command_hint" + sort -u "$ORACLE_BASELINE_TMP" + } >"$ORACLE_BASELINE_OUT" +} + +oracle_expectation_summary() { + printf '%s' "$ORACLE_EXPECTED_FAILURES expected failures, " + printf '%s' "$ORACLE_UNEXPECTED_FAILURES unexpected failures, " + printf '%s' "$ORACLE_UNEXPECTED_PASSES unexpected passes" +} + +oracle_expectations_ok() { + [[ "$ORACLE_UNEXPECTED_FAILURES" -eq 0 && "$ORACLE_UNEXPECTED_PASSES" -eq 0 ]] +} new file mode 100644 --- /dev/null +++ b/tests/oracle/patterns-expected-fails.txt @@ -0,0 +1,4 @@ +# Expected-fail baseline for tests/oracle/patterns-sweep.sh +# Format: <case-id> <last-observed-outcome> +# Regenerate with: +# BASELINE_OUT=tests/oracle/patterns-expected-fails.txt tests/oracle/patterns-sweep.sh --- a/tests/oracle/patterns-sweep.sh +++ b/tests/oracle/patterns-sweep.sh @@ -33,9 +33,14 @@ MAX_CASES="${MAX_CASES:-0}" MAX_DIFFS="${MAX_DIFFS:-20}" CASE_REGEX="${CASE_REGEX:-}" LIST_MISMATCHES="${LIST_MISMATCHES:-0}" +EXPECTED_FAILS="${EXPECTED_FAILS:-$ROOT/tests/oracle/patterns-expected-fails.txt}" +BASELINE_OUT="${BASELINE_OUT:-}" TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/jerboa-semgrep-patterns.XXXXXX")" trap 'rm -rf "$TMPDIR"' EXIT +source "$ROOT/tests/oracle/expected-fails.sh" +oracle_expectations_init "$TMPDIR" "$EXPECTED_FAILS" "$BASELINE_OUT" + normalize_json() { "$SCHEME" --libdirs "$LIBDIRS" --script "$NORMALIZE" | sort } @@ -93,6 +98,7 @@ mismatch=0 skipped=0 current_error=0 jerboa_error=0 +unexpected_failure=0 diffs_shown=0 IFS=' ' read -r -a lang_dirs <<<"$PATTERN_LANGS" @@ -156,6 +162,7 @@ for lang_dir in "${lang_dirs[@]}"; do if ! SEMGREP_SEND_METRICS=off run_json "$lang_dir.$base.current" "$current_json" \ "$SEMGREP_CURRENT" scan --quiet --json --config "$config" "$target"; then current_error=$((current_error + 1)) + oracle_record_failure "$case_id" "current-error" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "ERROR current $case_id" fi @@ -174,6 +181,7 @@ for lang_dir in "${lang_dirs[@]}"; do "$SCHEME" --libdirs "$LIBDIRS" --script "$ROOT/bin/semgrep.ss" \ scan --json --config "$config" "$target"; then jerboa_error=$((jerboa_error + 1)) + oracle_record_failure "$case_id" "jerboa-error" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "ERROR jerboa $case_id" fi @@ -189,9 +197,11 @@ for lang_dir in "${lang_dirs[@]}"; do normalize_json <"$jerboa_json" >"$jerboa_norm" if diff -u "$current_norm" "$jerboa_norm" >"$TMPDIR/$lang_dir.$base.diff"; then + oracle_record_pass "$case_id" || unexpected_failure=1 pass=$((pass + 1)) else mismatch=$((mismatch + 1)) + oracle_record_failure "$case_id" "mismatch" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "MISMATCH $case_id" fi @@ -204,5 +214,9 @@ for lang_dir in "${lang_dirs[@]}"; do done < <(find "$pattern_dir" -maxdepth 1 -type f -name '*.sgrep' | sort) done -echo "patterns-sweep: $pass passed, $mismatch mismatched, $jerboa_error jerboa errors, $current_error current errors, $skipped skipped, $total compared" -[[ "$mismatch" -eq 0 && "$jerboa_error" -eq 0 && "$current_error" -eq 0 ]] +oracle_write_baseline \ + "tests/oracle/patterns-sweep.sh" \ + "BASELINE_OUT=$EXPECTED_FAILS tests/oracle/patterns-sweep.sh" + +echo "patterns-sweep: $pass passed, $mismatch mismatched, $jerboa_error jerboa errors, $current_error current errors, $skipped skipped, $total compared, $(oracle_expectation_summary)" +[[ "$unexpected_failure" -eq 0 ]] && oracle_expectations_ok new file mode 100644 --- /dev/null +++ b/tests/oracle/upstream-expected-fails.txt @@ -0,0 +1,7 @@ +# Expected-fail baseline for tests/oracle/upstream-sweep.sh +# Format: <case-id> <last-observed-outcome> +# Regenerate with: +# BASELINE_OUT=tests/oracle/upstream-expected-fails.txt tests/oracle/upstream-sweep.sh +anywhere_global current-error +anywhere_include current-error +anywhere_metavar current-error --- a/tests/oracle/upstream-sweep.sh +++ b/tests/oracle/upstream-sweep.sh @@ -31,9 +31,14 @@ MAX_CASES="${MAX_CASES:-0}" MAX_DIFFS="${MAX_DIFFS:-20}" CASE_REGEX="${CASE_REGEX:-}" LIST_MISMATCHES="${LIST_MISMATCHES:-0}" +EXPECTED_FAILS="${EXPECTED_FAILS:-$ROOT/tests/oracle/upstream-expected-fails.txt}" +BASELINE_OUT="${BASELINE_OUT:-}" TMPDIR="$(mktemp -d "${TMPDIR:-/tmp}/jerboa-semgrep-upstream.XXXXXX")" trap 'rm -rf "$TMPDIR"' EXIT +source "$ROOT/tests/oracle/expected-fails.sh" +oracle_expectations_init "$TMPDIR" "$EXPECTED_FAILS" "$BASELINE_OUT" + normalize_json() { "$SCHEME" --libdirs "$LIBDIRS" --script "$NORMALIZE" | sort } @@ -78,6 +83,7 @@ pass=0 mismatch=0 current_error=0 jerboa_error=0 +unexpected_failure=0 diffs_shown=0 while IFS= read -r rule; do @@ -105,6 +111,7 @@ while IFS= read -r rule; do if ! SEMGREP_SEND_METRICS=off run_json "$case_name.current" "$current_json" \ "$SEMGREP_CURRENT" scan --quiet --json --config "$rule" "$target"; then current_error=$((current_error + 1)) + oracle_record_failure "$case_name" "current-error" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "ERROR current $case_name" fi @@ -123,6 +130,7 @@ while IFS= read -r rule; do "$SCHEME" --libdirs "$LIBDIRS" --script "$ROOT/bin/semgrep.ss" \ scan --json --config "$rule" "$target"; then jerboa_error=$((jerboa_error + 1)) + oracle_record_failure "$case_name" "jerboa-error" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "ERROR jerboa $case_name" fi @@ -138,9 +146,11 @@ while IFS= read -r rule; do normalize_json <"$jerboa_json" >"$jerboa_norm" if diff -u "$current_norm" "$jerboa_norm" >"$TMPDIR/$case_name.diff"; then + oracle_record_pass "$case_name" || unexpected_failure=1 pass=$((pass + 1)) else mismatch=$((mismatch + 1)) + oracle_record_failure "$case_name" "mismatch" || unexpected_failure=1 if [[ "$LIST_MISMATCHES" != "0" ]]; then echo "MISMATCH $case_name" fi @@ -152,5 +162,9 @@ while IFS= read -r rule; do fi done < <(find "$RULE_DIR" -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) | sort) -echo "upstream-sweep: $pass passed, $mismatch mismatched, $jerboa_error jerboa errors, $current_error current errors, $total compared" -[[ "$mismatch" -eq 0 && "$jerboa_error" -eq 0 && "$current_error" -eq 0 ]] +oracle_write_baseline \ + "tests/oracle/upstream-sweep.sh" \ + "BASELINE_OUT=$EXPECTED_FAILS tests/oracle/upstream-sweep.sh" + +echo "upstream-sweep: $pass passed, $mismatch mismatched, $jerboa_error jerboa errors, $current_error current errors, $total compared, $(oracle_expectation_summary)" +[[ "$unexpected_failure" -eq 0 ]] && oracle_expectations_ok