Add Chez upstream security sync runner
ober
8145589ba9f61de066d739eacb4581ae601cded7
--- a/data/anti-patterns.sexp +++ b/data/anti-patterns.sexp @@ -4142,4 +4142,26 @@ "jerboa_verify" "ranking equivalence test" "focused accumulator benchmark" + "jerboa_security_scan")) + (("advice" + . + "At map-open time recompute every pruning summary from validated postings and document metadata, require exact equality, and reject the entire disposable index on any mismatch. Use only mathematically safe upper bounds under live corpus statistics, preserve deterministic tie rules, and regression-test pruned results against an unpruned reference across replacements and tombstones.") + ("avoid" + . + "Do not use persisted block maxima, score upper bounds, or skip metadata to omit candidates after checking only offsets and a file checksum. A corrupt value or writer bug can understate the bound and silently remove valid results.") + ("id" . "trusting-unvalidated-pruning-metadata") + ("kinds" "module" "test" "security") + ("pattern" + . + "(?:block|max|upper).*(?:bound|summary).*(?:skip|prun)(?!.*validat)") + ("severity" . "high") + ("tags" "search" "block-max" "mmap" "validation" "upper-bound" + "correctness") + ("title" + . + "Do Not Trust Persisted Pruning Bounds Without Recomputing Them") + ("tools" + "jerboa_verify" + "corrupt-index regression" + "ranking equivalence test" "jerboa_security_scan"))) new file mode 100755 --- /dev/null +++ b/scripts/sync-chez-upstream-security.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +# +# Use Codex in batch mode to inspect upstream cisco/ChezScheme for changes +# worth carrying into Jerboa's vendored Chez tree, apply those changes, and +# verify the full Jerboa build/test gate. +# +# This script is intentionally repo-local. Run it from anywhere inside this +# checkout; it only edits this repository. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: scripts/sync-chez-upstream-security.sh [options] + +Options: + --model MODEL Codex model to use. Default: $CODEX_MODEL or CLI default. + --codex PATH Codex executable. Default: $CODEX_BIN or codex. + --allow-dirty Permit running with existing uncommitted changes. + --skip-final-gate Do not run the post-Codex local verification gate. + --docker Always run make docker-build in the final gate. + --no-docker Do not run make docker-build in the final gate. + -h, --help Show this help. + +Environment: + CODEX_MODEL Default model when --model is omitted. + CODEX_BIN Default codex executable when --codex is omitted. + CODEX_EXTRA_ARGS Extra arguments appended to codex exec. + REPORT_DIR Output directory for logs. Default: dist/chez-upstream-sync-<timestamp>. + RUN_DOCKER auto, 1, or 0. Default: auto (Linux only). + ALLOW_DIRTY 1 permits an existing dirty tree. + SKIP_FINAL_GATE 1 skips post-Codex verification. + +What the final gate runs: + make binary + make test-all + make verify + make docker-build # Linux in auto mode, or when --docker is used +EOF +} + +root="$(git rev-parse --show-toplevel 2>/dev/null || true)" +if [ -z "$root" ]; then + echo "error: not inside a git repository" >&2 + exit 2 +fi + +cd "$root" + +codex_bin="${CODEX_BIN:-codex}" +model="${CODEX_MODEL:-}" +allow_dirty="${ALLOW_DIRTY:-0}" +skip_final_gate="${SKIP_FINAL_GATE:-0}" +run_docker="${RUN_DOCKER:-auto}" + +while [ $# -gt 0 ]; do + case "$1" in + --model) + [ $# -ge 2 ] || { echo "error: --model requires an argument" >&2; exit 2; } + model="$2" + shift 2 + ;; + --codex) + [ $# -ge 2 ] || { echo "error: --codex requires an argument" >&2; exit 2; } + codex_bin="$2" + shift 2 + ;; + --allow-dirty) + allow_dirty=1 + shift + ;; + --skip-final-gate) + skip_final_gate=1 + shift + ;; + --docker) + run_docker=1 + shift + ;; + --no-docker) + run_docker=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if ! command -v "$codex_bin" >/dev/null 2>&1; then + echo "error: codex executable not found: $codex_bin" >&2 + exit 127 +fi + +if [ "$allow_dirty" != 1 ] && [ -n "$(git status --porcelain)" ]; then + echo "error: working tree has uncommitted changes; commit/stash or pass --allow-dirty" >&2 + git status --short + exit 1 +fi + +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +report_dir="${REPORT_DIR:-dist/chez-upstream-sync-$timestamp}" +mkdir -p "$report_dir" + +codex_log="$report_dir/codex.log" +final_log="$report_dir/final-gate.log" +prompt_file="$report_dir/prompt.txt" + +cat > "$prompt_file" <<EOF +You are running inside this Jerboa checkout: $root. Stay inside this repository. + +Goal: +Sync Jerboa's vendored Chez Scheme with the official upstream +https://github.com/cisco/ChezScheme enough to carry any upstream security, +hardening, memory-safety, parser/deserializer, FFI, Windows loader, build +integrity, or reliability fixes that Jerboa needs. + +Important repository facts: +- The vendored tree is vendor/ChezScheme. +- vendor/ChezScheme is a flat snapshot, not a git submodule. +- Do not run the vendored sync-upstream Makefile target against this checkout; + it will use the Jerboa repo git metadata. +- Use vendor/ChezScheme/UPSTREAM.md and docs/chez-fork.md to identify the + current upstream base and local divergence model. +- Work only in this repo. Do not touch sibling repos. + +Required workflow: +1. Create a temporary upstream checkout outside the repo, such as under /tmp. + Fetch https://github.com/cisco/ChezScheme. +2. Compare upstream main against the base recorded in vendor/ChezScheme/UPSTREAM.md. +3. Identify upstream commits not present in vendor/ChezScheme. Prioritize changes + that are security-relevant or could affect correctness in hostile input, + native/FFI, loader, parser, FASL/vfasl, bytevector, build, or platform code. +4. Apply only the changes Jerboa should carry. Keep edits scoped. +5. If you apply upstream changes, update Jerboa's Chez documentation or patch + records when needed so future maintainers can understand what was imported. +6. Run verification and fix failures until clean: + - make binary + - make test-all + - make verify + On Linux, if Docker is available, also run: + - make docker-build +7. End with a concise report listing: + - upstream base and upstream head reviewed + - commits considered + - commits applied or explicitly skipped, with reasons + - files changed + - verification commands and pass/fail status + +Do not claim success unless the verification commands you ran actually passed. +EOF + +codex_args=( + exec + --cd "$root" + --sandbox danger-full-access + --ask-for-approval never + --search +) + +if [ -n "$model" ]; then + codex_args+=(--model "$model") +fi + +if [ -n "${CODEX_EXTRA_ARGS:-}" ]; then + # shellcheck disable=SC2206 + extra_args=(${CODEX_EXTRA_ARGS}) + codex_args+=("${extra_args[@]}") +fi + +echo "==> Report directory: $report_dir" +echo "==> Running Codex batch review/apply pass..." +"$codex_bin" "${codex_args[@]}" - < "$prompt_file" 2>&1 | tee "$codex_log" + +if [ "$skip_final_gate" = 1 ]; then + echo "==> Skipping final local verification gate by request." + exit 0 +fi + +{ + echo "==> Final local verification gate started at $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "==> make binary" + make binary + echo "==> make test-all" + make test-all + echo "==> make verify" + make verify + + case "$run_docker" in + 1) + echo "==> make docker-build" + make docker-build + ;; + 0) + echo "==> docker-build skipped" + ;; + auto) + if [ "$(uname -s)" = "Linux" ]; then + echo "==> make docker-build (Linux auto mode)" + make docker-build + else + echo "==> docker-build skipped (host is $(uname -s); use --docker to force)" + fi + ;; + *) + echo "error: RUN_DOCKER must be auto, 1, or 0" >&2 + exit 2 + ;; + esac + + echo "==> Final local verification gate passed at $(date -u +%Y-%m-%dT%H:%M:%SZ)" +} 2>&1 | tee "$final_log" + +echo "==> Done. Logs:" +echo " $codex_log" +echo " $final_log"