Security hardening and release readiness
ober
eae75c1094261050ec9e4c38c07eec99d06ec2a1
new file mode 100644 --- /dev/null +++ b/.build.yml @@ -0,0 +1,25 @@ +image: debian/stable +packages: + - build-essential + - ca-certificates + - curl + - git + - ripgrep +environment: + JERBOA_VERSION: v0.2.3 +sources: + - https://git.sr.ht/~lisp/jerboa-semgrep + - https://git.sr.ht/~lisp/jerboa-treesitter +tasks: + - install-jerbuild: | + curl -fsSL "https://github.com/jerboa-lang/jerboa/releases/download/${JERBOA_VERSION}/jerbuild-linux-x86_64" -o "$HOME/jerbuild" + chmod +x "$HOME/jerbuild" + - build-treesitter: | + cd jerboa-treesitter + JERBUILD="$HOME/jerbuild" make build + - verify: | + cd jerboa-semgrep + JERBUILD="$HOME/jerbuild" make verify + - release-evidence: | + cd jerboa-semgrep + JERBUILD="$HOME/jerbuild" make release-evidence new file mode 100644 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,39 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install build tools + run: sudo apt-get update && sudo apt-get install -y build-essential curl ca-certificates git ripgrep + + - name: Install jerbuild + run: | + set -eux + curl -fsSL "https://github.com/jerboa-lang/jerboa/releases/download/${JERBOA_VERSION}/jerbuild-linux-x86_64" -o /usr/local/bin/jerbuild + chmod +x /usr/local/bin/jerbuild + env: + JERBOA_VERSION: v0.2.3 + + - name: Build jerboa-treesitter + run: | + set -eux + git clone https://git.sr.ht/~lisp/jerboa-treesitter ../jerboa-treesitter + JERBUILD=/usr/local/bin/jerbuild make -C ../jerboa-treesitter build + + - name: Verify + run: JERBUILD=/usr/local/bin/jerbuild make verify + + - name: Release evidence + run: JERBUILD=/usr/local/bin/jerbuild make release-evidence new file mode 100644 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Native and compiled artifacts +*.so +*.wpo +*.boot +*.o +*.dylib +*.tarm64osx +*.tx86_64osx +*.ta6le +*.ta6fb + +# Local oracle/test output +tests/oracle/out/ +tests/oracle/tmp/ +dist/ +*.log + +# Local tools and editor caches +.claude/ +*~ +\#* +.#* new file mode 100644 --- /dev/null +++ b/.jerboa/security.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "repo": "jerboa-semgrep", + "extends": ["jerboa:cli", "jerboa:parser", "jerboa:untrusted-input"], + "classification": "local Semgrep-compatible scanner for untrusted rule configs and source targets", + "license": "Apache-2.0", + "paths": { + "production": ["bin/**/*.ss", "src/**/*.ss", "lib/**/*.sls", "Makefile"], + "tests": ["tests/**", "support/import-check.ss"], + "generated": ["dist/**", "tests/oracle/out/**", "tests/oracle/tmp/**", "*.so", "*.dylib", "*.wpo", "*.o"], + "vendor": ["vendor/**", "third_party/**", "node_modules/**"], + "docs": ["README.md", "SECURITY.md", "docs/**", "*.md"] + }, + "capabilities": { + "network": "none", + "ffi": "none in this repository; parsing FFI is isolated in the sibling jerboa-treesitter dependency", + "filesystemRead": "rule configs, .semgrepignore files, source targets, and stdin targets", + "filesystemWrite": "only --autofix writes modified target files when requested by the caller", + "shell": "none in runtime code", + "hostileInput": "Semgrep YAML/JSON rule configs, ignore files, source code targets, parser outputs, and stdin" + }, + "policy": { + "failOn": ["critical", "high"], + "imports": { "directChezscheme": "deny in source, bin, support, scripts, and tests" }, + "ffi": { "allowed": false }, + "process": { "allowed": false }, + "network": { "allowed": false }, + "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false }, + "resourceLimits": { + "configCharsEnv": "JSEMGREP_MAX_CONFIG_CHARS", + "targetCharsEnv": "JSEMGREP_MAX_TARGET_CHARS" + } + }, + "releaseGates": [ + "make security", + "make test", + "make verify", + "make dependency-evidence", + "make sbom", + "make reproducibility-report", + "make target-evidence", + "make release-evidence" + ], + "targetProof": { + "requiredForProduction": true, + "environment": "JSEMGREP_TARGET_PROOF_FILE", + "requireEnvironment": "JSEMGREP_REQUIRE_TARGET_PROOF", + "sensitiveArtifactPolicy": "no-source-code-findings-or-private-paths" + }, + "suppressions": [] +} --- a/HANDOFF_OPUS_4_8.md +++ b/HANDOFF_OPUS_4_8.md @@ -1,9 +1,9 @@ # Opus 4.8 Handoff: jerboa-semgrep Semgrep Parity Date: 2026-06-02 (continuation) -Workspace: `/Users/user/mine/jerboa-semgrep` -Sibling upstream Semgrep checkout: `/Users/user/mine/semgrep` -Packaged Semgrep oracle: `/Users/user/.local/bin/semgrep` +Workspace: this repository +Sibling upstream Semgrep checkout: sibling `semgrep` +Packaged Semgrep oracle: `semgrep` on `PATH` or `SEMGREP_CURRENT` Branch: `main` — structural migrations below are consolidated; the former `dart-wip` branch has been folded in. @@ -358,12 +358,12 @@ a real taint engine (4); approximating (2)/(3) in the regex matcher risks the ```sh make test make oracle -SEMGREP_CURRENT=/Users/user/.local/bin/semgrep \ -UPSTREAM_RULE_DIR=/Users/user/mine/semgrep/tests/tainting_rules/go \ +SEMGREP_CURRENT=/path/to/semgrep \ +UPSTREAM_RULE_DIR=../semgrep/tests/tainting_rules/go \ LIST_MISMATCHES=1 MAX_DIFFS=0 tests/oracle/upstream-sweep.sh # swap go/php ``` -Broad sweep (slow): `UPSTREAM_RULE_DIR=/Users/user/mine/semgrep/tests/rules`. +Broad sweep (slow): `UPSTREAM_RULE_DIR=../semgrep/tests/rules`. Harness notes: foreground `sleep` and background `while/until` loops are blocked/unstable; run a sweep via a single `run_in_background` invocation and sample `*.norm` counts for throughput; do NOT `make build` while a sweep runs new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. --- a/Makefile +++ b/Makefile @@ -1,61 +1,159 @@ -JERBOA_HOME ?= $(HOME)/mine/jerboa -JERBOA_TREESITTER_HOME ?= $(HOME)/mine/jerboa-treesitter -HOST_SCHEME ?= $(firstword $(wildcard $(JERBOA_HOME)/.chez/bin/scheme $(HOME)/mine/ChezScheme/tarm64osx/bin/tarm64osx/scheme) scheme) -SCHEME ?= $(HOST_SCHEME) -JERBUILD ?= $(if $(wildcard $(JERBOA_HOME)/jerbuild),$(JERBOA_HOME)/jerbuild,$(SCHEME) --libdirs "$(JERBOA_HOME)/lib" --script "$(JERBOA_HOME)/jerbuild.ss") -LIBDIRS = lib:$(JERBOA_TREESITTER_HOME)/lib:$(JERBOA_HOME)/lib +JERBOA_VERSION ?= v0.2.3 +JERBOA_TOOL_DIR ?= ../jerboa/dist +ifeq ($(origin JERBUILD),undefined) +ifneq ($(wildcard $(JERBOA_TOOL_DIR)/jerbuild),) +JERBUILD := $(JERBOA_TOOL_DIR)/jerbuild +else +JERBUILD := jerbuild +endif +endif +JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) +ifeq ($(JH),) +$(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) +endif + +JERBOA_TREESITTER_HOME ?= ../jerboa-treesitter +LIBDIRS := lib:$(JERBOA_TREESITTER_HOME)/lib:$(JH)/lib +JEXEC := $(JERBUILD) exec --libdirs "$(LIBDIRS)" +EVIDENCE_DIR ?= dist/release-evidence +SBOM_DIR ?= dist/sbom +REPRO_DIR ?= dist/reproducibility +TARGET_EVIDENCE_DIR ?= dist/target-evidence + +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Darwin) + SHLIB_EXT := dylib + LD_VAR := DYLD_LIBRARY_PATH +else + SHLIB_EXT := so + LD_VAR := LD_LIBRARY_PATH +endif PATTERN_CASE_REGEX ?= ^(ac_matching_dots|ac_matching_dots1|ac_matching_explosion|ac_matching_explosion1|ac_matching_explosion2|ac_matching_explosion3|ac_matching_free|ac_matching_free1|ac_matching_if|ac_matching_if1|ac_matching_mvars|ac_matching_mvars1|ac_matching_mvars2|aliasing_attribute|assoc_matching_bug|assoc_matching_dots|assoc_matching_dots1|assoc_matching_explosion|assoc_matching_explosion1|assoc_matching_explosion2|assoc_matching_explosion3|assoc_matching_free|assoc_matching_free1|assoc_matching_mvars|assoc_matching_mvars1|constprop_dataflow|constprop_used_twice|cp_concat|cp_exception|cp_label|cp_label1|cp_method_call|cp_python_mult_string|cp_python_mult_string1|cp_rlval|cp_string_mvar|cp_strings|cp_with|cp_yield|deep_cond|deep_expr_operator|deep_expr_vs_statement|deep_exprstmt|deep_fake_dataflow|deep_try|dict_ellipsis|dots_attribute|dots_expr_plus|dots_expr_plus_string|dots_fstring|dots_fstring_with_match_stmt|dots_inherit|dots_list|dots_nested_stmts|dots_stmts|equivalence_constant_propagation|equivalence_f_string|equivalence_f_string_2|equivalence_f_string_3|equivalence_f_string_4|equivalence_interpolated_str|equivalence_interpolated_str2|equivalence_keyword_args|equivalence_naming_import|import_metavar|import_negatives|import_negatives2|imports|index_tuple|less_attributes|less_catch|less_catch_multiple|less_inherits|less_typehint|less_typehint2|matching_if_expr|metavar_anno|metavar_anno_fqn|metavar_class_def|metavar_dict|metavar_equality_param_vs_use|metavar_equality_var|metavar_func_def|metavar_stmt|metavar_typed|misc_attributes|misc_block_import|misc_class_regression|misc_dots_stmts|misc_except_matching|misc_faketok2|misc_if1|misc_metavar_vs_fstring|misc_naming_bug2|misc_paren|misc_range_statements|misc_regression1|misc_regression2|misc_stmts1|misc_tuple2|misc_with_parens|multi_import|multi_qualified_wildcard|pip614-extended-decorator-grammer1|regexp_string_backref|set_vs_dict|set_vs_dict2|standalone_decorator|standalone_decorator_single_arg|static_method|stmts_to_fields|tuple_subscript_ellipsis|tuple_subscript_ellipsis_with_match_stmt|unordered_metavar2|untyped_vs_typed|wildcard_qualified)$$ JS_PATTERN_CASE_REGEX ?= ^(ac_matching_bitor|aliasing_require|caching_deep|caching_deep_metavar|caching_different_depth|caching_flat|caching_nested|cp_conditional|cp_dataflow|cp_implicit_conversion|cp_incrdecr|cp_label|cp_throw|cp_undeclared|cp_undeclared1|deep_cond|deep_deep_stmt1|deep_deep_stmt2|deep_expr_and_ellipsis|deep_expr_vs_statement|deep_expr_xml|deep_stmt|deep_stmt2|deep_vardef|dots_field_chaining|dots_importfrom|dots_jsx_attr|dots_jsx_body|dots_jsx_body2|dots_template_literals|ellipsis_metavar_tagged_template|ellipsis_metavar_template|equivalence_aliasing_import|equivalence_import_require|equivalence_import_variations|equivalence_import_variations2|equivalence_import_variations3|equivalence_import_variations4|equivalence_keyword_args|equivalence_varlet|infer_const|infer_const_1|infer_const_regexp|equivalence_constant_propagation2|equivalence_constant_propagation3|less_class_complex|less_class_complex_2|less_inheritance|less_xml_attr|less_xml_body|metavar_class_fields|metavar_equality_vardef_vs_use|metavar_import|metavar_import2|metavar_import3|metavar_importfrom|metavar_importfrom2|metavar_jsx_val|metavar_stmt_or_expr|metavar_template|metavar_typed_bool|metavar_typed_literal|metavar_xml_attr|misc_arrow|misc_arrow2|misc_asi_pattern|misc_catch_unbound|misc_decorators_method|misc_deep_expr_stmt|misc_empty_body|misc_export|misc_method_chain_vs_stmt|misc_object_directly|misc_tagged_template_literal|object_duplicate_key|object_numeric_key|partial_catch|partial_decorators_arg_method|partial_decorators_method|partial_function|partial_lambda_or_func|partial_single_field|regexp_fieldname|switch_case_pattern|template_string_w_metavar|typed_metavar_assign|unordered_destructuring|unordered_fields|unordered_fields2|unordered_metavar|unordered_xml_attr)$$ -.PHONY: all build generate test oracle oracle-upstream patterns-oracle patterns-js-oracle clean +.PHONY: all build generate test malformed-corpus import-check security audit verify dependency-evidence sbom reproducibility-report target-evidence release-evidence oracle oracle-upstream patterns-oracle patterns-js-oracle clean clean-generated help +.DEFAULT_GOAL := help all: build build: - $(JERBUILD) src lib --force + $(JERBUILD) transpile src lib --force generate: - $(SCHEME) --libdirs "$(JERBOA_HOME)/lib" \ - --script scripts/generate-lang-registry.ss \ + $(JEXEC) scripts/generate-lang-registry.ss \ "../semgrep/cli/src/semgrep/semgrep_interfaces/lang.json" \ > src/semgrep/schema/lang.ss test: build - JERBOA_TREE_SITTER_LIB=$(JERBOA_TREESITTER_HOME) \ - DYLD_LIBRARY_PATH=$(JERBOA_TREESITTER_HOME):$$DYLD_LIBRARY_PATH \ - LD_LIBRARY_PATH=$(JERBOA_TREESITTER_HOME):$$LD_LIBRARY_PATH \ - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/smoke.ss + @JERBOA_TREE_SITTER_LIB=$(JERBOA_TREESITTER_HOME) \ + $(LD_VAR)=$(JERBOA_TREESITTER_HOME):$${$(LD_VAR)} \ + $(JEXEC) tests/smoke.ss + +malformed-corpus: build support/malformed-corpus-evidence.ss + @JERBOA_TREE_SITTER_LIB=$(JERBOA_TREESITTER_HOME) \ + $(LD_VAR)=$(JERBOA_TREESITTER_HOME):$${$(LD_VAR)} \ + $(JEXEC) support/malformed-corpus-evidence.ss + +import-check: build + @JERBOA_TREE_SITTER_LIB=$(JERBOA_TREESITTER_HOME) \ + $(LD_VAR)=$(JERBOA_TREESITTER_HOME):$${$(LD_VAR)} \ + $(JEXEC) support/import-check.ss + +security: clean-generated scripts/security-check.sh + scripts/security-check.sh + +audit: security + @test -d "$(JERBOA_TREESITTER_HOME)" || { echo "missing jerboa-treesitter checkout: $(JERBOA_TREESITTER_HOME)" >&2; exit 1; } + @test -f "$(JERBOA_TREESITTER_HOME)/lib/tree-sitter/tree-sitter.sls" || { echo "missing jerboa-treesitter transpiled library; build the sibling dependency first" >&2; exit 1; } + @test -f "$(JERBOA_TREESITTER_HOME)/libjerboa_tree_sitter.$(SHLIB_EXT)" || { echo "missing jerboa-treesitter native shim: libjerboa_tree_sitter.$(SHLIB_EXT)" >&2; exit 1; } + @echo "==> jerbuild" + @$(JERBUILD) --version + @echo "==> jerboa-treesitter native linkage" + @if command -v otool >/dev/null 2>&1; then \ + otool -L "$(JERBOA_TREESITTER_HOME)/libjerboa_tree_sitter.$(SHLIB_EXT)"; \ + elif command -v ldd >/dev/null 2>&1; then \ + ldd "$(JERBOA_TREESITTER_HOME)/libjerboa_tree_sitter.$(SHLIB_EXT)"; \ + else \ + echo "No otool or ldd available for linkage audit."; \ + fi + +target-evidence: scripts/target-evidence.sh + @REPO_ROOT=. TARGET_EVIDENCE_DIR="$(TARGET_EVIDENCE_DIR)" sh scripts/target-evidence.sh + +verify: audit import-check malformed-corpus test target-evidence + +dependency-evidence: + $(MAKE) -C "$(JERBOA_TREESITTER_HOME)" JERBUILD="$(JERBUILD)" release-evidence + +sbom: dependency-evidence + REPO_ROOT="$(CURDIR)" SBOM_DIR="$(SBOM_DIR)" JERBUILD="$(JERBUILD)" JERBOA_VERSION="$(JERBOA_VERSION)" JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" sh scripts/sbom.sh + +reproducibility-report: + REPO_ROOT="$(CURDIR)" REPRO_DIR="$(REPRO_DIR)" MAKE="$(MAKE)" sh scripts/reproducibility-report.sh + +release-evidence: verify sbom reproducibility-report + rm -rf $(EVIDENCE_DIR) + mkdir -p $(EVIDENCE_DIR) + { git rev-parse HEAD 2>/dev/null || true; } > $(EVIDENCE_DIR)/git-commit.txt + git status --short > $(EVIDENCE_DIR)/git-status.txt + git diff --stat > $(EVIDENCE_DIR)/diff-stat.txt + { printf 'JERBOA_VERSION=%s\n' '$(JERBOA_VERSION)'; "$(JERBUILD)" --version; if "$(JERBUILD)" --jerboa-home >/dev/null 2>&1; then echo "jerboa_home_status=present"; else echo "jerboa_home_status=missing"; fi; printf 'jerboa_treesitter_home_name=%s\n' "$$(basename '$(JERBOA_TREESITTER_HOME)')"; uname -srm; } > $(EVIDENCE_DIR)/build-env.txt + $(MAKE) security > $(EVIDENCE_DIR)/security.log 2>&1 + $(MAKE) audit > $(EVIDENCE_DIR)/audit.log 2>&1 + $(MAKE) import-check > $(EVIDENCE_DIR)/import-check.log 2>&1 + $(MAKE) malformed-corpus > $(EVIDENCE_DIR)/malformed-corpus.log 2>&1 + $(MAKE) test > $(EVIDENCE_DIR)/test.log 2>&1 + $(MAKE) target-evidence > $(EVIDENCE_DIR)/target-evidence.log 2>&1 + { find . -type f -not -path './.git/*' -not -path './dist/*' -not -path './tests/oracle/out/*' -not -path './tests/oracle/tmp/*' -not -name '*.so' -not -name '*.dylib' -not -name '*.wpo' -not -name '*.wp.so' -not -name '*.o' -not -name '*.a' | LC_ALL=C sort | while IFS= read -r f; do shasum -a 256 "$$f"; done; } > $(EVIDENCE_DIR)/source-sha256.txt + rm -rf $(EVIDENCE_DIR)/sbom $(EVIDENCE_DIR)/reproducibility $(EVIDENCE_DIR)/target-evidence + cp -R $(SBOM_DIR) $(EVIDENCE_DIR)/sbom + cp -R $(REPRO_DIR) $(EVIDENCE_DIR)/reproducibility + cp -R $(TARGET_EVIDENCE_DIR) $(EVIDENCE_DIR)/target-evidence + grep -q '^target_evidence_status=present$$' "$(EVIDENCE_DIR)/target-evidence/status.txt" + test "$$(grep '^status=' "$(EVIDENCE_DIR)/reproducibility/report.txt" | cut -d= -f2)" = "match" oracle: build - JERBOA_HOME="$(JERBOA_HOME)" \ JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ - SCHEME="$(SCHEME)" \ LIBDIRS="$(LIBDIRS)" \ tests/oracle/run.sh oracle-upstream: build - JERBOA_HOME="$(JERBOA_HOME)" \ JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ - SCHEME="$(SCHEME)" \ LIBDIRS="$(LIBDIRS)" \ tests/oracle/upstream-sweep.sh patterns-oracle: build - JERBOA_HOME="$(JERBOA_HOME)" \ JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ - SCHEME="$(SCHEME)" \ LIBDIRS="$(LIBDIRS)" \ CASE_REGEX="$(PATTERN_CASE_REGEX)" \ tests/oracle/patterns-sweep.sh patterns-js-oracle: build - JERBOA_HOME="$(JERBOA_HOME)" \ JERBOA_TREESITTER_HOME="$(JERBOA_TREESITTER_HOME)" \ - SCHEME="$(SCHEME)" \ LIBDIRS="$(LIBDIRS)" \ PATTERN_LANGS=js \ CASE_REGEX="$(JS_PATTERN_CASE_REGEX)" \ tests/oracle/patterns-sweep.sh clean: - rm -rf lib + find . \( -name '*.so' -o -name '*.dylib' -o -name '*.wpo' -o -name '*.wp.so' -o -name '*.o' -o -name '*.a' \) -delete 2>/dev/null || true + rm -rf dist tests/oracle/out tests/oracle/tmp + +clean-generated: + find . \( -name '*.so' -o -name '*.dylib' -o -name '*.wpo' -o -name '*.wp.so' -o -name '*.o' -o -name '*.a' \) -delete 2>/dev/null || true + rm -rf tests/oracle/out tests/oracle/tmp + +help: + @echo "jerboa-semgrep" + @echo "" + @echo "Targets:" + @echo " make build Transpile src/*.ss into tracked lib/*.sls" + @echo " make test Run smoke and regression tests" + @echo " make malformed-corpus Run hostile rule/source corpus evidence" + @echo " make import-check Verify public modules load" + @echo " make security Static release/security checks" + @echo " make audit Security plus jerboa-treesitter dependency audit" + @echo " make verify Audit, import check, and tests" + @echo " make target-evidence Record target-only production blockers" + @echo " make release-evidence Generate dist/release-evidence" + @echo " make clean Remove native/test/evidence artifacts; preserve lib/" --- a/README.md +++ b/README.md @@ -45,13 +45,47 @@ Unsupported Semgrep rule forms fail explicitly instead of pretending to be compatible. Tree-sitter grammars are provided by `jerboa-treesitter`, which uses statically linked grammars only. -## Build And Test +## Build, Test, And Release Gates ```sh make test ``` -`JERBOA_TREESITTER_HOME` defaults to `~/mine/jerboa-treesitter`. +`JERBOA_TREESITTER_HOME` defaults to `../jerboa-treesitter`. That sibling +checkout must be built first because parser FFI is isolated there. + +Run the release/security gates: + +```sh +make security +make malformed-corpus +make verify +make target-evidence +make release-evidence +``` + +`make verify` runs static security checks, dependency audit, import checks, the +512-case hostile rule/source corpus, and the smoke/regression test suite. +`make release-evidence` writes +`dist/release-evidence`, including Semgrep SBOM/source hashes, refreshed sibling +`jerboa-treesitter` grammar/runtime evidence, and a repeated-transpile +reproducibility report for tracked generated Scheme artifacts, normalized +Jerbuild hash metadata, repeated corpus output, and target scanner proof +status. + +Production support requires marker-complete target proof through +`JSEMGREP_TARGET_PROOF_FILE` with `JSEMGREP_REQUIRE_TARGET_PROOF=1`; without it, +target-only oracle, compatibility, review, autofix, and output-redaction gates +remain blocked. + +Input size caps are enforced before rule configs and targets are fully read: + +- `JSEMGREP_MAX_CONFIG_CHARS`, default `4194304` +- `JSEMGREP_MAX_TARGET_CHARS`, default `8388608` + +This repository is expected to remain network-free, process-execution-free, and +free of direct Scheme FFI. Native parser linkage belongs in `jerboa-treesitter` +and is audited by `make audit`. Regenerate the Semgrep language registry from the sibling checkout: @@ -65,7 +99,7 @@ Run the focused compatibility oracle against a current Semgrep binary: make oracle ``` -`make oracle` looks for `/Users/user/mine/semgrep/bin/semgrep`, then `semgrep` -on `PATH`. Set `SEMGREP_CURRENT=/path/to/semgrep` to override it. If no current +`make oracle` looks for a sibling Semgrep checkout binary, then `semgrep` on +`PATH`. Set `SEMGREP_CURRENT=/path/to/semgrep` to override it. If no current Semgrep binary is available, the oracle skips cleanly. The current local baseline passes 42/42 normalized oracle cases against packaged Semgrep 1.164.0. new file mode 100644 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,44 @@ +# Security Policy + +`jerboa-semgrep` is a local Semgrep-compatible scanner. Treat rule configs, +ignore files, source targets, and stdin as untrusted input. + +Production claims require the release plan and production-readiness tracker +gates to be complete for this repository. + +## Supported Status + +No public production-support commitment exists yet. Security-sensitive releases +must be cut from a clean checkout after: + +- `git status --short` shows only intentional release changes. +- `make security`, `make verify`, and `make release-evidence` pass. +- `make target-evidence` records reviewed target proof through + `JSEMGREP_TARGET_PROOF_FILE` with `JSEMGREP_REQUIRE_TARGET_PROOF=1`. +- The working tree secret scan is clean, and history has been reviewed or + intentionally reset before first public release. +- `jerboa-treesitter` has been built and audited as the parser FFI boundary. +- Accepted dependency or parser risks are documented in release evidence. +- Upstream Semgrep oracle/compatibility evidence, autofix safety, output + sensitivity, and external parser/scanner review are represented by + marker-complete target proof. + +## Security Expectations + +- Runtime source must remain network-free and process-execution-free. +- Direct Scheme FFI is not allowed in this repository. +- Parser FFI must stay isolated in `jerboa-treesitter`. +- Rule config and target reads must keep the documented character limits. +- `--autofix` is the only intentional target-file write path. +- Finding output can include source snippets and metavariable captures; treat it + as potentially sensitive. +- Do not commit secrets, private keys, tokens, production `.env` files, + operational hostnames, customer/user data, or private infrastructure details. +- Target proof files must not contain source code, finding payloads, private + paths, hostnames, credentials, or customer data. + +## Reporting + +Before public release, report issues privately to the repository owner. After +public release, replace this section with a dedicated advisory contact, +supported versions, and disclosure window. --- a/SEMGREP_JERBOA_IMPLEMENTATION.md +++ b/SEMGREP_JERBOA_IMPLEMENTATION.md @@ -4,9 +4,9 @@ Inventory date: 2026-06-03 Local repositories inspected: -- `jerboa-semgrep`: `/Users/user/mine/jerboa-semgrep` -- Upstream Semgrep: `/Users/user/mine/semgrep`, `develop` at `238ad257b` -- Jerboa libraries: `/Users/user/mine/jerboa-*` +- `jerboa-semgrep`: this repository +- Upstream Semgrep: sibling `semgrep` checkout, `develop` at `238ad257b` +- Jerboa libraries: sibling `jerboa-*` checkouts This document records what the current port already contains, what the Opus-created branches contributed, and the practical path to a complete @@ -425,8 +425,7 @@ The harness should support: - Expected-pass, expected-fail, and expected-difference manifests. - Per-language filtering. - Per-feature filtering. -- Differential comparison against upstream Semgrep from - `/Users/user/mine/semgrep`. +- Differential comparison against upstream Semgrep from a sibling checkout. - Stable JSON output normalization before comparison. ## Progress Log @@ -865,7 +864,7 @@ Fixture tests: Differential tests: -- Run upstream Semgrep from `/Users/user/mine/semgrep`. +- Run upstream Semgrep from a sibling checkout. - Run Jerboa Semgrep on the same config/target. - Normalize nondeterministic fields such as timing, path roots, version strings, and ordering where upstream allows it. new file mode 100644 --- /dev/null +++ b/docs/parser-hardening.md @@ -0,0 +1,37 @@ +# Parser Hardening + +`jerboa-semgrep` treats rule configs, targets, and stdin as hostile input. + +The scanner enforces character-count caps before fully materializing these +inputs: + +- `JSEMGREP_MAX_CONFIG_CHARS`, default `4194304` +- `JSEMGREP_MAX_TARGET_CHARS`, default `8388608` + +Both values must be positive integers. Invalid or unset values fall back to the +defaults. + +Path-based readers reject empty paths and paths containing NUL bytes. Stdin is +read through the same target-size limiter used for target files. + +Structural parse failures are limited to expected parser and structural matcher +conditions. Broad catch-all guards are rejected by `scripts/security-check.sh` +so unrelated implementation errors are not silently converted into misses. + +`make malformed-corpus` runs 512 deterministic hostile rule/source cases: + +- valid rule config parsing for supported pattern forms, taint rules, path + filters, generic rules, JavaScript rules, and fix metadata; +- scanner cases that must produce findings across regex, structural, + `pattern-either`, `patterns`, generic, and path-filtered rules; +- expected-reject malformed rule configs for missing required fields, + unsupported languages, unsupported modes, invalid `patterns`, invalid taint + shapes, and malformed path filters; +- malformed and bounded target sources that must not escape as unexpected + crashes. + +Security marker values consumed by `scripts/security-check.sh`: + +- `malformed_corpus_status: documented` +- `malformed_corpus_cases_minimum: 512` +- `malformed_corpus_scope_status: rules-scan-malformed-source-bounded-source` new file mode 100644 --- /dev/null +++ b/docs/release-evidence.md @@ -0,0 +1,55 @@ +# Release Evidence + +`make release-evidence` writes a release evidence bundle to +`dist/release-evidence`. + +The bundle includes: + +- git commit, status, and diff summary, +- Jerboa and tree-sitter dependency environment metadata, +- security, audit, import-check, malformed corpus, and test logs, +- source-file SHA-256 digests, +- `sbom/`, with Semgrep source/release-input hashes plus copied + `jerboa-treesitter` runtime, grammar, native-linkage, SBOM, and + reproducibility evidence, +- `reproducibility/report.txt`, comparing repeated transpilation output for + tracked `lib/*.sls` files, normalized `src/.jerbuild-hashes` metadata, and + repeated malformed corpus output. +- `target-evidence/status.txt` and `target-evidence/proof-template.txt`, which + record target-only production blockers for oracle compatibility, longer + corpus/soak evidence, dependency proof, external review, autofix safety, and + output redaction. + +Before publishing, review `git-status.txt` and `diff-stat.txt` to confirm only +intentional source, generated Scheme artifacts, docs, metadata, and release +evidence are present. + +The evidence target preserves tracked `lib/*.sls` files because this repository +commits transpiled Scheme artifacts as part of its source distribution. + +`make release-evidence` refreshes sibling `jerboa-treesitter` release evidence +before recording this repository's SBOM, so grammar/runtime provenance remains +attached to the Semgrep evidence bundle. + +`make target-evidence` records local status by default. Production scanner +support requires `JSEMGREP_REQUIRE_TARGET_PROOF=1` and +`JSEMGREP_TARGET_PROOF_FILE=/path/to/proof.txt`; missing or marker-incomplete +proof fails closed before release evidence is accepted. + +Required target proof markers: + +```text +target_evidence_status: local-release-proof-recorded +target_scanner_proof_status: not-run +production_scanner_status: blocked-not-run +upstream_semgrep_oracle_status=reviewed +long_compatibility_corpus_status=reviewed +target_tree_sitter_dependency_status=reviewed +external_parser_scanner_review_status=reviewed +autofix_safety_review_status=reviewed +source_output_redaction_policy_status=reviewed +sensitive_artifact_policy=no-source-code-findings-or-private-paths +``` + +Proof files must not contain source code, finding payloads, private paths, +hostnames, credentials, or customer data. new file mode 100644 --- /dev/null +++ b/docs/threat-model.md @@ -0,0 +1,36 @@ +# Threat Model + +`jerboa-semgrep` is a local command-line scanner. It accepts untrusted rule +configuration, ignore files, source targets, and stdin content, then emits +findings in JSON, SARIF, or text formats. It is not a daemon and must not open +network listeners or make outbound network requests. + +## Assets + +- Local source files and repository metadata being scanned. +- Finding output, including snippets and metavariable captures. +- Target files when `--autofix` is explicitly requested. +- The host process running the scanner. + +## Trust Boundaries + +- Semgrep config files are untrusted input. +- Source targets and stdin are untrusted input. +- `.semgrepignore` files are untrusted input. +- Tree-sitter parsing crosses into the sibling `jerboa-treesitter` native shim. +- `--autofix` crosses from read-only scanning to caller-approved file writes. + +## Security Requirements + +- Runtime source must remain network-free and process-execution-free. +- This repository must not use Scheme FFI directly. +- File reads must reject invalid paths and apply documented character limits. +- Parser failures for unsupported or malformed structural patterns must fail + closed without hiding unrelated runtime errors. +- Test cleanup must use taint-aware filesystem helpers. +- `make verify` must pass before release evidence is generated. + +## Accepted Residual Risk + +Language parsing depends on `jerboa-treesitter`. Production releases must audit +that sibling dependency and include linkage evidence for its native shim. new file mode 100644 --- /dev/null +++ b/docs/tree-sitter-boundary.md @@ -0,0 +1,19 @@ +# Tree-Sitter Boundary + +This repository does not use Scheme FFI directly. Parser FFI is isolated in the +sibling `jerboa-treesitter` repository. + +`jerboa-semgrep` expects: + +- a sibling checkout at `../jerboa-treesitter`, or an explicit + `JERBOA_TREESITTER_HOME`, +- transpiled Scheme modules under `$(JERBOA_TREESITTER_HOME)/lib`, and +- `libjerboa_tree_sitter.dylib` on macOS or `libjerboa_tree_sitter.so` on + Linux. + +The `make audit` target verifies those files and records the native linkage +with `otool -L` or `ldd` when available. CI builds `jerboa-treesitter` before +verifying this repository. + +Tree-sitter grammars are statically compiled into the sibling shim. New grammar +support should be added there first, with its own native-boundary audit. --- a/lib/semgrep/cli.sls +++ b/lib/semgrep/cli.sls @@ -13,7 +13,11 @@ (semgrep lang) (semgrep scan) (semgrep result) (semgrep rule) (semgrep rule parse-rule) (semgrep fix) (semgrep output json) (semgrep output sarif) - (semgrep output text)) + (semgrep output text) + (only + (semgrep targeting path-filter) + read-target-file-string + read-target-port-string)) (def (usage) (display "usage: semgrep scan --config RULES.yml [--lang LANG] [--severity LEVEL] [--include GLOB] [--exclude GLOB] TARGET\n") @@ -155,15 +159,13 @@ (def (read-semgrepignore dir) (let ([path (sg-path-join dir ".semgrepignore")]) (if (file-exists? path) - (call-with-input-file - path - (lambda (port) - (let loop ([lines (split-lines (get-string-all port))] - [acc '()]) - (cond - [(null? lines) (reverse acc)] - [(ignore-line? (car lines)) (loop (cdr lines) acc)] - [else (loop (cdr lines) (cons (car lines) acc))])))) + (let loop ([lines (split-lines + (read-target-file-string path))] + [acc '()]) + (cond + [(null? lines) (reverse acc)] + [(ignore-line? (car lines)) (loop (cdr lines) acc)] + [else (loop (cdr lines) (cons (car lines) acc))])) '()))) (def (ignore-pattern-matches? pattern relative name) (or (glob-match? pattern relative) @@ -300,7 +302,7 @@ rules language "<stdin>" - (get-string-all (current-input-port))) + (read-target-port-string (current-input-port) "<stdin>")) (scan-file rules language target-path)))) (def (findings-for-path path findings) (filter @@ -310,9 +312,7 @@ (unless (string=? target "-") (let ([target-findings (findings-for-path target findings)]) (unless (null? target-findings) - (let ([source (call-with-input-file - target - (lambda (port) (get-string-all port)))]) + (let ([source (read-target-file-string target)]) (write-file-string target (apply-fixes-to-string source target-findings))))))) --- a/lib/semgrep/rule/parse-rule.sls +++ b/lib/semgrep/rule/parse-rule.sls @@ -11,10 +11,52 @@ \x31;- partition make-date make-time meta atom?) (except (jerboa prelude) meta atom?) (std text yaml) (semgrep lang) (semgrep rule)) + (define default-max-config-chars (* 4 1024 1024)) + (define (configured-positive-int name default) + (let ([value (getenv name)]) + (if value + (let ([n (string->number value)]) + (if (and n (integer? n) (> n 0)) n default)) + default))) + (define max-config-chars + (configured-positive-int + "JSEMGREP_MAX_CONFIG_CHARS" + default-max-config-chars)) + (def (string-contains-nul? s) + (let ([len (string-length s)]) + (let loop ([i 0]) + (and (< i len) + (or (char=? (string-ref s i) (integer->char 0)) + (loop (+ i 1))))))) + (def (check-config-path! path) + (unless (and (string? path) (> (string-length path) 0)) + (error 'parse-config-file + "path must be a non-empty string" + path)) + (when (string-contains-nul? path) + (error 'parse-config-file "path contains NUL byte" path)) + path) + (def (read-port-limited port limit who path) + (let loop ([chars '()] [count 0]) + (let ([ch (read-char port)]) + (cond + [(eof-object? ch) (list->string (reverse chars))] + [(>= count limit) + (error who + "input exceeds configured character limit" + path + limit)] + [else (loop (cons ch chars) (+ count 1))])))) (def (read-config-file-string path) - (call-with-input-file - path - (lambda (port) (get-string-all port)))) + (let ([checked-path (check-config-path! path)]) + (call-with-input-file + checked-path + (lambda (port) + (read-port-limited + port + max-config-chars + 'parse-config-file + checked-path))))) (def (split-lines source) (let ([len (string-length source)]) (let loop ([i 0] [start 0] [acc '()]) --- a/lib/semgrep/scan.sls +++ b/lib/semgrep/scan.sls @@ -17706,8 +17706,16 @@ (unless (null? xs) (node-close! (structural-match-node (car xs))) (loop (cdr xs))))) + (def (condition-who=? e who) + (and (who-condition? e) (eq? (condition-who e) who))) + (def (expected-structural-text-failure? e) + (or (condition-who=? e 'parse-target-string) + (condition-who=? e 'structural-pattern-matches) + (condition-who=? + e + 'structural-pattern-matches-with-bindings))) (def (structural-text-matches? language pattern source) - (guard (e [#t #f]) + (guard (e [(expected-structural-text-failure? e) #f]) (let* ([parse-result (parse-target-string language source)] [root (parse-result-root parse-result)] [matches (structural-pattern-matches @@ -17720,7 +17728,7 @@ (tree-close! (parse-result-tree parse-result)) matched?))) (def (structural-text-root-matches? language pattern source) - (guard (e [#t #f]) + (guard (e [(expected-structural-text-failure? e) #f]) (let* ([trimmed (string-trim source)] [parse-result (parse-target-string language trimmed)] [root (parse-result-root parse-result)]