Security hardening and release readiness
ober
3f13b19bf61b6dcbc4c939e99453037a0befda04
new file mode 100644 --- /dev/null +++ b/.build.yml @@ -0,0 +1,19 @@ +image: debian/stable +packages: + - build-essential + - ca-certificates + - curl + - ripgrep + - libpcre2-dev +environment: + JERBOA_VERSION: v0.2.3 +tasks: + - install-jerbuild: | + curl -fsSL "https://github.com/jerboa-lang/jerboa/releases/download/${JERBOA_VERSION}/jerbuild-linux-x86_64" -o "$HOME/jerbuild" + chmod +x "$HOME/jerbuild" + - verify: | + cd jerboa-sed + JERBUILD="$HOME/jerbuild" make verify + - release-evidence: | + cd jerboa-sed + JERBUILD="$HOME/jerbuild" make release-evidence new file mode 100644 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install build tools + run: sudo apt-get update && sudo apt-get install -y build-essential curl ca-certificates ripgrep libpcre2-dev + + - name: Install jerbuild + run: | + set -eux + curl -fsSL "https://github.com/jerboa-lang/jerboa/releases/download/${JERBOA_VERSION}/jerbuild-linux-x86_64" -o /usr/local/bin/jerbuild + chmod +x /usr/local/bin/jerbuild + env: + JERBOA_VERSION: v0.2.3 + + - name: Verify + run: make verify + + - name: Release evidence + run: make release-evidence new file mode 100644 --- /dev/null +++ b/.github/workflows/security-baseline.yml @@ -0,0 +1,35 @@ +name: Security Baseline + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + baseline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Required release files + run: | + set -eu + test -f LICENSE + test -f SECURITY.md + test -f .gitignore + find . -maxdepth 1 -iname "README*" -type f | grep -q . + + - name: High-confidence secret scan + run: | + set -eu + pattern="(BEGIN (RSA|OPENSSH|EC|DSA|PRIVATE) KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16})" + matches="$(git grep -n -I -E "$pattern" -- . ":!*.png" ":!*.jpg" ":!*.jpeg" ":!*.gif" ":!*.so" ":!*.dylib" ":!*.o" ":!*.a" ":!*.boot" ":!*.tar.gz" || true)" + if [ -n "$matches" ]; then + echo "$matches" + echo "High-confidence secret pattern found." + exit 1 + fi --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ *.so +*.wpo +*.o +*.a +*.boot jsed +/.build/ +/dist/ +.jsed-test-out +.jsed-test-file +.jsed-test-file.bak +.jsed-test-script.sed --- a/.jerboa/security.json +++ b/.jerboa/security.json @@ -1,22 +1,49 @@ { "version": 1, "repo": "jerboa-sed", - "extends": ["jerboa:cli", "jerboa:parser"], + "extends": ["jerboa:cli", "jerboa:parser", "jerboa:ffi"], + "classification": "hostile-input parser, local CLI, and PCRE2 FFI consumer", + "license": "Apache-2.0", "paths": { - "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.ss", "src/**/*.sls", "bin/**", "Makefile"], + "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "Makefile"], "tests": ["test/**", "tests/**", "**/*-test.ss"], - "generated": ["build/**", "dist/**", "target/**"], - "vendor": ["vendor/**", "third_party/**"], + "generated": ["build/**", "dist/**", "target/**", "jsed", "*.so", "*.dylib", "*.wpo"], + "vendor": ["vendor/**", "third_party/**", "node_modules/**"], "docs": ["README.md", "docs/**", "*.md", "AGENTS.md"] }, + "capabilities": { + "network": "none", + "ffi": "PCRE2 dynamic library only, isolated in lib/sed/pcre2.sls", + "shell": "disabled by default; JSED_ALLOW_SHELL=1 enables sed e and s///e", + "scriptFileIo": "disabled by default; JSED_ALLOW_SCRIPT_FILE_IO=1 enables sed r/R/w/W and s///w", + "cliFileIo": "input files, -f scripts, and explicit -i edits are supported with taint/path checks" + }, + "resourceLimits": { + "JSED_MAX_SCRIPT_CHARS": "maximum sed script text length; default 1048576", + "JSED_MAX_INPUT_LINE_CHARS": "maximum input line length; default 1048576" + }, "policy": { "failOn": ["critical", "high"], - "imports": { "directChezscheme": "deny" }, - "ffi": { "allowed": false }, - "process": { "shellInterpolation": "deny" }, - "network": { "requireTimeouts": false }, - "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false }, - "parser": { "requireLimits": true } + "imports": { "directChezscheme": "deny except targeted PCRE2 FFI boundary" }, + "ffi": { "allowedOnlyIn": ["lib/sed/pcre2.sls"] }, + "process": { "allowedOnlyBehind": "JSED_ALLOW_SHELL=1" }, + "network": { "allowed": false }, + "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false } + }, + "releaseGates": [ + "make security", + "make test", + "make verify", + "make sbom", + "make reproducibility-report", + "make target-evidence", + "make release-evidence" + ], + "targetProof": { + "requiredForProduction": true, + "environment": "JSED_TARGET_PROOF_FILE", + "requireEnvironment": "JSED_REQUIRE_TARGET_PROOF", + "sensitiveArtifactPolicy": "no-input-text-output-captures-private-paths-or-host-data" }, "suppressions": [] } --- a/AGENTS.md +++ b/AGENTS.md @@ -340,11 +340,10 @@ improved versions of the above. When working in a Jerboa project, **ONLY modify files in the current repo** unless the user explicitly names another path. Common sibling repos that exist but must NOT be touched without explicit instruction: -- `~/mine/jerboa-emacs` — **NEVER touch**. Another model owns it. -- `~/mine/jerboa-mcp` — Only modify when user explicitly says to work there. -- `~/mine/jerboa-shell` — Only modify when user explicitly says to work there. -- `~/mine/gerbil-mcp` — **NEVER touch**. Deprecated. -- `~/mine/gerbil-orig` — Read-only reference for upstream Gerbil. Never modify. +- sibling editor integration repo — **NEVER touch** unless explicitly assigned. +- sibling MCP integration repo — Only modify when user explicitly says to work there. +- sibling shell implementation repo — Only modify when user explicitly says to work there. +- non-current-project archives — Never modify from this repository. If a user instruction mentions a file path, use EXACTLY that path. Do not substitute a similar-looking path from another repo. new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. --- a/Makefile +++ b/Makefile @@ -1,16 +1,26 @@ -# jerbuild bundles Chez Scheme + the jerboa stdlib under ~/.cache/jerbuild/, -# so building jsed needs only `jerbuild` + a C compiler — no jerboa source -# checkout and no separately-built Chez. -JERBUILD ?= jerbuild +JERBOA_VERSION ?= v0.2.3 +JERBOA_TOOL_DIR ?= ../jerboa/dist +ifeq ($(origin JERBUILD),undefined) +ifneq ($(wildcard $(JERBOA_TOOL_DIR)/jerbuild),) +JERBUILD := $(JERBOA_TOOL_DIR)/jerbuild +else +JERBUILD := jerbuild +endif +endif JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) ifeq ($(JH),) $(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) endif LIBDIRS := --libdirs lib:$(JH)/lib +JEXEC := $(JERBUILD) exec $(LIBDIRS) JSED_BIN := jsed +EVIDENCE_DIR ?= dist/release-evidence +SBOM_DIR ?= dist/sbom +REPRO_DIR ?= dist/reproducibility +TARGET_EVIDENCE_DIR ?= dist/target-evidence -.PHONY: all build binary run test clean install +.PHONY: all build binary run test test-parser parser-corpus test-cli-policy import-check clean-generated security audit verify sbom reproducibility-report target-evidence release-evidence clean install all: binary @@ -25,17 +35,90 @@ run: binary ./$(JSED_BIN) $(ARGS) clean: - find lib \( -name '*.so' -o -name '*.wpo' \) -delete 2>/dev/null || true - rm -f $(JSED_BIN) + find lib \( -name '*.so' -o -name '*.wpo' -o -name '*.o' \) -delete 2>/dev/null || true + rm -rf dist + rm -f $(JSED_BIN) .jsed-test-out .jsed-test-file .jsed-test-file.bak .jsed-test-script.sed install: binary install -d $(HOME)/.local/bin install -m 0755 $(JSED_BIN) $(HOME)/.local/bin/$(JSED_BIN) -test: binary +test-parser: + @$(JEXEC) tests/test-parser.ss + +parser-corpus: support/parser-corpus-evidence.ss + @$(JEXEC) support/parser-corpus-evidence.ss + +test-cli-policy: binary + @echo "--- shell disabled by default ---"; test -z "$$(echo ignored | ./$(JSED_BIN) -n 'e printf hacked')" + @echo "--- shell opt-in ---"; echo ignored | JSED_ALLOW_SHELL=1 ./$(JSED_BIN) -n 'e printf ok' | grep -qx ok + @echo "--- sandbox overrides shell opt-in ---"; test -z "$$(echo ignored | JSED_ALLOW_SHELL=1 ./$(JSED_BIN) --sandbox -n 'e printf hacked')" + @echo "--- script file I/O disabled by default ---"; rm -f .jsed-test-out; echo ok | ./$(JSED_BIN) 'w .jsed-test-out' >/dev/null; test ! -e .jsed-test-out + @echo "--- script file I/O opt-in ---"; rm -f .jsed-test-out; echo ok | JSED_ALLOW_SCRIPT_FILE_IO=1 ./$(JSED_BIN) 'w .jsed-test-out' >/dev/null; grep -qx ok .jsed-test-out; rm -f .jsed-test-out + @echo "--- script size cap ---"; printf 'pp' > .jsed-test-script.sed; if JSED_MAX_SCRIPT_CHARS=1 ./$(JSED_BIN) -f .jsed-test-script.sed >/tmp/jsed-size.out 2>/tmp/jsed-size.err; then cat /tmp/jsed-size.out /tmp/jsed-size.err; rm -f .jsed-test-script.sed; exit 1; fi; rm -f .jsed-test-script.sed + +test: test-parser binary @echo "--- s/// ---"; echo "hello world" | ./$(JSED_BIN) 's/hello/goodbye/' @echo "--- = (line #) ---"; printf "a\nb\nc\n" | ./$(JSED_BIN) '=' @echo "--- 2d ---"; printf "a\nb\nc\n" | ./$(JSED_BIN) '2d' @echo "--- s///g ---"; echo "aaa" | ./$(JSED_BIN) 's/a/b/g' @echo "--- hold space ---"; printf "1\n2\n3\n" | ./$(JSED_BIN) -n '1h;2{x;p};3{x;p}' + $(MAKE) test-cli-policy @echo "All tests passed." + +import-check: + @$(JEXEC) support/import-check.ss + +clean-generated: + find lib \( -name '*.so' -o -name '*.wpo' -o -name '*.o' \) -delete 2>/dev/null || true + rm -f $(JSED_BIN) .jsed-test-out .jsed-test-file .jsed-test-file.bak .jsed-test-script.sed + +security: clean-generated import-check scripts/security-check.sh + scripts/security-check.sh + +audit: security + @if command -v pcre2-config >/dev/null 2>&1; then \ + printf 'PCRE2 version: '; pcre2-config --version; \ + elif command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libpcre2-8; then \ + printf 'PCRE2 version: '; pkg-config --modversion libpcre2-8; \ + else \ + echo 'PCRE2 version: not found locally; runtime must provide libpcre2-8'; \ + fi + +target-evidence: scripts/target-evidence.sh + @REPO_ROOT=. TARGET_EVIDENCE_DIR="$(TARGET_EVIDENCE_DIR)" sh scripts/target-evidence.sh + +verify: audit parser-corpus test target-evidence + +sbom: + REPO_ROOT="$(CURDIR)" SBOM_DIR="$(SBOM_DIR)" JERBUILD="$(JERBUILD)" JERBOA_VERSION="$(JERBOA_VERSION)" sh scripts/sbom.sh + +reproducibility-report: + REPO_ROOT="$(CURDIR)" REPRO_DIR="$(REPRO_DIR)" MAKE="$(MAKE)" sh scripts/reproducibility-report.sh + +release-evidence: verify + $(MAKE) reproducibility-report + $(MAKE) sbom + rm -rf $(EVIDENCE_DIR) + mkdir -p $(EVIDENCE_DIR) + rm -f $(JSED_BIN) + { git rev-parse HEAD 2>/dev/null || true; } > $(EVIDENCE_DIR)/git-commit.txt + git status --short > $(EVIDENCE_DIR)/git-status.txt + git diff --stat > $(EVIDENCE_DIR)/diff-stat.txt + { printf 'JERBOA_VERSION=%s\n' '$(JERBOA_VERSION)'; "$(JERBUILD)" --version; if "$(JERBUILD)" --jerboa-home >/dev/null 2>&1; then echo "jerboa_home_status=present"; else echo "jerboa_home_status=missing"; fi; uname -srm; env | LC_ALL=C sort | grep -E '^(JSED_ALLOW_SCRIPT_FILE_IO|JSED_ALLOW_SHELL|JSED_MAX_INPUT_LINE_CHARS|JSED_MAX_SCRIPT_CHARS)=' || true; } > $(EVIDENCE_DIR)/build-env.txt + $(MAKE) security > $(EVIDENCE_DIR)/security.log 2>&1 + $(MAKE) audit > $(EVIDENCE_DIR)/audit.log 2>&1 + $(MAKE) parser-corpus > $(EVIDENCE_DIR)/parser-corpus.log 2>&1 + $(MAKE) test > $(EVIDENCE_DIR)/test.log 2>&1 + $(MAKE) target-evidence > $(EVIDENCE_DIR)/target-evidence.log 2>&1 + echo "hello world" | ./$(JSED_BIN) 's/world/jerboa/' > $(EVIDENCE_DIR)/smoke.txt + shasum -a 256 ./$(JSED_BIN) > $(EVIDENCE_DIR)/binary-sha256.txt + { find . -type f -not -path './.git/*' -not -path './dist/*' -not -path './.build/*' -not -name '$(JSED_BIN)' -not -name '*.so' -not -name '*.wpo' -not -name '*.wp.so' -not -name '*-main.c' -not -name 'petite_boot.h' -not -name 'scheme_boot.h' -not -name 'jsed_program.h' | LC_ALL=C sort | while IFS= read -r f; do shasum -a 256 "$$f"; done; } > $(EVIDENCE_DIR)/source-sha256.txt + rm -rf $(EVIDENCE_DIR)/sbom $(EVIDENCE_DIR)/reproducibility $(EVIDENCE_DIR)/target-evidence + cp -R $(SBOM_DIR) $(EVIDENCE_DIR)/sbom + cp -R $(REPRO_DIR) $(EVIDENCE_DIR)/reproducibility + cp -R $(TARGET_EVIDENCE_DIR) $(EVIDENCE_DIR)/target-evidence + grep -q '^target_evidence_status=present$$' "$(EVIDENCE_DIR)/target-evidence/status.txt" + test "$$(grep '^status=' "$(EVIDENCE_DIR)/reproducibility/report.txt" | cut -d= -f2)" = "match" + sh scripts/sanitize-evidence.sh "$(EVIDENCE_DIR)" + rm -f $(JSED_BIN) .jsed-test-out .jsed-test-file .jsed-test-file.bak .jsed-test-script.sed --- a/README.md +++ b/README.md @@ -1 +1,43 @@ # jerboa-sed + +Jerboa `sed` implementation and standalone `jsed` CLI. + +## Security Status + +This repository is not a production-supported public release yet. It now has a +local release gate: + +```sh +make verify +make target-evidence +make release-evidence +``` +`make release-evidence` records SBOM/toolchain/PCRE2 evidence and a two-pass +binary reproducibility report using a cleanly prewarmed generated-library input +snapshot for the generated `jsed` binary. It also captures the deterministic +512-case parser corpus evidence and target sed proof status. Production support +requires marker-complete target proof through `JSED_TARGET_PROOF_FILE` with +`JSED_REQUIRE_TARGET_PROOF=1`. + +Secure defaults: + +- sed script shell execution is disabled unless `JSED_ALLOW_SHELL=1` is set; +- sed script file reads/writes (`r`, `R`, `w`, `W`, `s///w`) are disabled unless + `JSED_ALLOW_SCRIPT_FILE_IO=1` is set; +- `--sandbox` disables both script shell and script file effects even when the + opt-in environment variables are set; +- script text and input records are capped by `JSED_MAX_SCRIPT_CHARS` and + `JSED_MAX_INPUT_LINE_CHARS`, both defaulting to 1 MiB. + +CLI input files, `-f` script files, and explicit `-i` in-place edits are treated +as caller intent and are still supported with path validation and taint checks. + +## Build And Test + +```sh +make parser-corpus +make test +make verify +``` + +`jsed` depends on the platform `libpcre2-8` runtime library. new file mode 100644 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,45 @@ +# Security Policy + +`jerboa-sed` is a local text-processing CLI and hostile-input parser. Treat it +as experimental until release evidence, history review, reproducibility checks, +SBOM/toolchain notes, and external parser/FFI review are complete. + +## Supported Status + +No public production-support commitment exists yet. + +Security-sensitive releases must be cut from a clean checkout after: + +- `make verify` and `make release-evidence` pass; +- `make sbom` and `make reproducibility-report` pass; +- `make target-evidence` records reviewed target proof through + `JSED_TARGET_PROOF_FILE` with `JSED_REQUIRE_TARGET_PROOF=1`; +- `scripts/security-check.sh` reports no generated artifacts, broad Chez imports, + unexpected FFI, unexpected process execution, raw deletes, or high-confidence + secrets; +- PCRE2 runtime version/linkage is recorded; +- history has been reviewed or intentionally reset before first public release. +- long sed/regex corpus, parser review, PCRE2 linkage, sandbox policy, and + script file-I/O review are represented by marker-complete target proof. + +## Default-Deny Policy + +- `e` and `s///e` do not execute shell commands unless `JSED_ALLOW_SHELL=1`. +- `r`, `R`, `w`, `W`, and `s///w` do not read/write script-named files unless + `JSED_ALLOW_SCRIPT_FILE_IO=1`. +- `--sandbox` disables script shell and script file effects even with those + environment opt-ins. +- `JSED_MAX_SCRIPT_CHARS` and `JSED_MAX_INPUT_LINE_CHARS` cap parser and input + memory use. Both default to 1 MiB. + +CLI input files, `-f` script files, and explicit `-i` in-place edits are caller +intent. They are not sandboxed, but paths are validated and taint-checked. + +Target proof files must not contain input text, output captures, private paths, +hostnames, credentials, or customer data. + +## Reporting + +Before public release, report issues privately to the repository owner. After +public release, replace this section with a dedicated advisory contact, +supported versions, and disclosure window. new file mode 100644 --- /dev/null +++ b/docs/ffi-boundary.md @@ -0,0 +1,15 @@ +# PCRE2 FFI Boundary + +`lib/sed/pcre2.sls` is the only FFI module in this repository. + +- Dynamic loading is limited to `libpcre2-8` platform names. +- The Scheme layer owns compiled regex handles and frees them after each address + or substitution use. +- Match-data handles are allocated for each search and freed on success and + no-match paths. +- Match-data allocation failures fail closed with an exception. +- The rest of the sed parser and engine does not import raw Chez FFI bindings. + +Release evidence should record the PCRE2 version and target-platform linkage. +Hosted CI installs `libpcre2-dev`; local release evidence records whatever +`pcre2-config` or `pkg-config` reports. new file mode 100644 --- /dev/null +++ b/docs/parser-hardening.md @@ -0,0 +1,26 @@ +# Parser Hardening + +The sed parser is treated as hostile-input code. + +- `JSED_MAX_SCRIPT_CHARS` caps inline and `-f` script text. The default is + 1 MiB. +- `JSED_MAX_INPUT_LINE_CHARS` caps input records read by the CLI. The default is + 1 MiB. +- Numeric sed addresses and widths reject values above a fixed safety limit. +- Parser tests cover substitutions, ranges, blocks, labels, malformed scripts, + oversized scripts, and oversized numeric addresses. +- `make parser-corpus` runs 512 deterministic parser cases covering + substitutions, addresses, blocks, file syntax, shell syntax, malformed + scripts, and bounded larger scripts. The release gate records this in + `dist/release-evidence/parser-corpus.log`. +- Shell execution (`e`, `s///e`) and script-file access (`r`, `R`, `w`, `W`, + `s///w`) are separate explicit opt-ins. + +Security marker values consumed by `scripts/security-check.sh`: + +- `parser_corpus_status: documented` +- `parser_corpus_cases_minimum: 512` +- `parser_corpus_scope_status: substitution-address-block-file-shell-malformed` + +Longer PCRE2 edge-case and large-line soak runs remain required before public +production claims. new file mode 100644 --- /dev/null +++ b/docs/release-evidence.md @@ -0,0 +1,58 @@ +# Release Evidence + +Run: + +```sh +make verify +make sbom +make reproducibility-report +make target-evidence +make release-evidence +``` + +`make release-evidence` writes `dist/release-evidence/` with: + +- git commit, status, and diff-stat; +- selected non-secret build environment values; +- security, import, audit, parser corpus, parser, CLI policy, and binary smoke + logs; +- PCRE2 version information when available; +- SHA-256 hashes for the generated `jsed` binary and repository source files; +- `sbom/`, a dependency/toolchain/PCRE2 manifest with source and release-input + hashes; +- `reproducibility/report.txt`, comparing two local `jsed` binary builds with + Jerboa fixed binary object-dir mode after a clean prewarm has produced a + stable generated-library input snapshot, plus prepared-input, + generated-input, parser corpus, and source manifests for the measured + comparison. +- `target-evidence/status.txt` and `target-evidence/proof-template.txt`, which + record target-only production blockers for long sed/regex corpus soak, parser + review, PCRE2 target linkage, sandbox policy, script file-I/O policy, and + external FFI review. + +Generated files under `dist/`, `.build/`, the root `jsed` binary, native object +files, boot files, and whole-program build artifacts are excluded from source +hashes and must remain untracked. + +`make target-evidence` records local status by default. Production `jsed` +support requires `JSED_REQUIRE_TARGET_PROOF=1` and +`JSED_TARGET_PROOF_FILE=/path/to/proof.txt`; missing or marker-incomplete proof +fails closed before release evidence is accepted. + +Required target proof markers: + +```text +target_evidence_status: local-release-proof-recorded +target_sed_proof_status: not-run +production_sed_status: blocked-not-run +long_sed_regex_corpus_status=reviewed +external_parser_review_status=reviewed +external_ffi_review_status=reviewed +pcre2_target_linkage_status=reviewed +sandbox_policy_review_status=reviewed +script_file_io_review_status=reviewed +sensitive_artifact_policy=no-input-text-output-captures-private-paths-or-host-data +``` + +Proof files must not contain input text, output captures, private paths, +hostnames, credentials, or customer data. new file mode 100644 --- /dev/null +++ b/docs/threat-model.md @@ -0,0 +1,34 @@ +# Threat Model + +`jerboa-sed` is a local command-line text transformer. Its hostile inputs are sed +program text, regular expressions, replacement text, input records, file names, +and environment-controlled policy toggles. + +## Assets + +- Local files named on the command line. +- Files referenced by sed scripts through `r`, `R`, `w`, `W`, and `s///w`. +- Command output generated by `e` and `s///e`. +- Process environment and current working directory. +- Availability of the calling process. + +## Boundaries + +- CLI input files, `-f` script files, and explicit `-i` in-place edits are user + intent and are allowed after path validation and taint checks. +- Script-driven file reads/writes are disabled unless + `JSED_ALLOW_SCRIPT_FILE_IO=1` is set. +- Script-driven shell execution is disabled unless `JSED_ALLOW_SHELL=1` is set. +- `--sandbox` disables script-driven file and shell effects even when the + environment opt-ins are set. +- PCRE2 is the only native boundary and is isolated in `lib/sed/pcre2.sls`. + +## Remaining Release Risks + +- PCRE2 regular-expression complexity can still consume CPU on hostile patterns. + Release candidates need longer malformed-regex and large-input corpus soaks. +- In-place editing intentionally modifies caller-selected files. It is not a + sandbox and should only be used on trusted paths. +- No public production-support commitment exists until release evidence, + history review, reproducibility comparison, SBOM/toolchain notes, and external + parser/FFI review are complete. --- a/lib/sed/ast.sls +++ b/lib/sed/ast.sls @@ -58,7 +58,7 @@ sed-flat-jump make-sed-flat-jump sed-flat-jump? sed-flat-jump-addr sed-flat-jump-target ;; Address extraction cmd-addr) - (import (chezscheme)) + (import (scheme)) ;;; Address primitives (define-record-type sed-addr-line (fields n)) --- a/lib/sed/engine.sls +++ b/lib/sed/engine.sls @@ -18,8 +18,12 @@ sed-program make-sed-program sed-program? sed-program-cmds sed-program-labels compile-program run-sed-cycle-from! - flush-append-queue!) - (import (except (chezscheme) compile-program) (sed ast) (sed pcre2)) + flush-append-queue! + close-open-write-files!) + (import (except (scheme) compile-program) + (only (std os env) getenv) + (only (std security taint) check-untainted!) + (sed ast) (sed pcre2)) ;;; Execution state (define-record-type sed-state @@ -47,6 +51,43 @@ (make-eq-hashtable) (make-hashtable string-hash string=?))) + (define (truthy-env? name) + (let ([value (getenv name)]) + (and value + (or (string=? value "1") + (string-ci=? value "true") + (string-ci=? value "yes"))))) + + (define (sed-shell-enabled?) + (truthy-env? "JSED_ALLOW_SHELL")) + + (define (sed-script-file-io-enabled? state) + (and (not (sed-state-sandbox? state)) + (truthy-env? "JSED_ALLOW_SCRIPT_FILE_IO"))) + + (define (string-contains-nul? s) + (let ([len (string-length s)]) + (let loop ([i 0]) + (and (< i len) + (or (char=? (string-ref s i) (integer->char 0)) + (loop (+ i 1))))))) + + (define (check-sed-path! who filename) + (unless (and (string? filename) (> (string-length filename) 0)) + (error who "path must be a non-empty string" filename)) + (when (string-contains-nul? filename) + (error who "path contains NUL byte" filename)) + (check-untainted! filename who) + filename) + + (define (check-sed-command! cmd) + (unless (and (string? cmd) (> (string-length cmd) 0)) + (error 'sed-shell "command must be a non-empty string" cmd)) + (when (string-contains-nul? cmd) + (error 'sed-shell "command contains NUL byte" cmd)) + (check-untainted! cmd 'sed-shell) + cmd) + ;;; Compiled program (define-record-type sed-program (fields cmds labels)) @@ -168,6 +209,13 @@ (when multiline? (set! flags (fxior flags PCRE2_MULTILINE))) (pcre2-compile pat flags))) + (define (with-compiled-rx pattern icase? multiline? extended? proc) + (let ([rx (compile-rx pattern icase? multiline? extended?)]) + (dynamic-wind + (lambda () #t) + (lambda () (proc rx)) + (lambda () (pcre2-free rx))))) + ;;; Address matching (define (addr-primitive-matches? addr state) (cond @@ -176,10 +224,11 @@ [(sed-addr-last? addr) (sed-state-last-line? state)] [(sed-addr-regex? addr) - (let ([rx (compile-rx (sed-addr-regex-pattern addr) - (sed-addr-regex-icase addr) #f - (sed-state-extended? state))]) - (pcre2-matches? rx (sed-state-pattern-space state)))] + (with-compiled-rx (sed-addr-regex-pattern addr) + (sed-addr-regex-icase addr) #f + (sed-state-extended? state) + (lambda (rx) + (pcre2-matches? rx (sed-state-pattern-space state))))] [(sed-addr-step? addr) (let ([first (sed-addr-step-first addr)] [step (sed-addr-step-step addr)] @@ -295,17 +344,18 @@ [nth (sed-s-flags-nth flags)] [icase? (sed-s-flags-icase? flags)] [mline? (sed-s-flags-multiline? flags)] - [rx (compile-rx (sed-cmd-s-pattern cmd) icase? mline? - (sed-state-extended? state))] [repl (sed-cmd-s-replacement cmd)] - [subject (sed-state-pattern-space state)] - [result (if global? - (gsub rx subject repl) - (sub-nth rx subject repl (or nth 1)))]) - (when result - (sed-state-pattern-space-set! state result) - (sed-state-sub-succeeded?-set! state #t)) - (if result #t #f))) + [subject (sed-state-pattern-space state)]) + (with-compiled-rx (sed-cmd-s-pattern cmd) icase? mline? + (sed-state-extended? state) + (lambda (rx) + (let ([result (if global? + (gsub rx subject repl) + (sub-nth rx subject repl (or nth 1)))]) + (when result + (sed-state-pattern-space-set! state result) + (sed-state-sub-succeeded?-set! state #t)) + (if result #t #f)))))) (define (sub-nth rx subject repl nth) (let loop ([start 0] [count 0]) @@ -425,8 +475,9 @@ ;;; File/shell helpers (define (write-to-file! state filename text) - (unless (sed-state-sandbox? state) - (let ([ht (sed-state-open-write-files state)]) + (when (sed-script-file-io-enabled? state) + (let* ([filename (check-sed-path! 'sed-script-file filename)] + [ht (sed-state-open-write-files state)]) (let ([port (or (hashtable-ref ht filename #f) (let ([p (open-file-output-port filename (file-options no-fail) @@ -438,6 +489,17 @@ (newline port) (flush-output-port port))))) + (define (close-open-write-files! state) + (let ([ht (sed-state-open-write-files state)]) + (let-values ([(keys vals) (hashtable-entries ht)]) + (let ([n (vector-length keys)]) + (let loop ([i 0]) + (when (< i n) + (guard (e [else (void)]) + (close-port (vector-ref vals i))) + (hashtable-delete! ht (vector-ref keys i)) + (loop (+ i 1)))))))) + (define (flush-append-queue! state out-port) (for-each (lambda (item) @@ -446,40 +508,47 @@ (put-string out-port item) (newline out-port)] [(and (pair? item) (eq? (car item) 'file)) - (guard (e [else (void)]) - (call-with-input-file (cdr item) - (lambda (p) - (let loop () - (let ([line (get-line p)]) - (unless (eof-object? line) - (put-string out-port line) - (newline out-port) - (loop)))))))] + (when (sed-script-file-io-enabled? state) + (let ([filename (check-sed-path! 'sed-script-file (cdr item))]) + (guard (e [else (void)]) + (call-with-input-file filename + (lambda (p) + (let loop () + (let ([line (get-line p)]) + (unless (eof-object? line) + (put-string out-port line) + (newline out-port) + (loop)))))))))] [(and (pair? item) (eq? (car item) 'file-line)) - (guard (e [else (void)]) - (call-with-input-file (cdr item) - (lambda (p) - (let ([line (get-line p)]) - (unless (eof-object? line) - (put-string out-port line) - (newline out-port))))))])) + (when (sed-script-file-io-enabled? state) + (let ([filename (check-sed-path! 'sed-script-file (cdr item))]) + (guard (e [else (void)]) + (call-with-input-file filename + (lambda (p) + (let ([line (get-line p)]) + (unless (eof-object? line) + (put-string out-port line) + (newline out-port))))))))])) (sed-state-append-queue state)) (sed-state-append-queue-set! state '())) (define (exec-shell! cmd) - (guard (e [else ""]) - (call-with-values - (lambda () - (open-process-ports - (string-append "/bin/sh -c " (shell-quote cmd)) - (buffer-mode block) - (make-transcoder (utf-8-codec)))) - (lambda (to-stdin from-stdout from-stderr pid) - (close-port to-stdin) - (let ([result (get-string-all from-stdout)]) - (close-port from-stdout) - (close-port from-stderr) - (if (eof-object? result) "" result)))))) + (if (not (sed-shell-enabled?)) + "" + (let ([cmd (check-sed-command! cmd)]) + (guard (e [else ""]) + (call-with-values + (lambda () + (open-process-ports + (string-append "/bin/sh -c " (shell-quote cmd)) + (buffer-mode block) + (make-transcoder (utf-8-codec)))) + (lambda (to-stdin from-stdout from-stderr pid) + (close-port to-stdin) + (let ([result (get-string-all from-stdout)]) + (close-port from-stdout) + (close-port from-stderr) + (if (eof-object? result) "" result)))))))) (define (shell-quote s) (string-append "'" (let loop ([i 0] [out (open-output-string)]) @@ -650,14 +719,14 @@ 'delete)))] ;; r [(sed-cmd-r? cmd) - (unless (sed-state-sandbox? state) + (when (sed-script-file-io-enabled? state) (sed-state-append-queue-set! state (append (sed-state-append-queue state) (list (cons 'file (sed-cmd-r-filename cmd)))))) 'continue] ;; R [(sed-cmd-R? cmd) - (unless (sed-state-sandbox? state) + (when (sed-script-file-io-enabled? state) (sed-state-append-queue-set! state