Harden DB release evidence privacy
ober
c4c14a4a203b3c1f6acda92a923b3d6e5185657f
--- a/.jerboa/security.json +++ b/.jerboa/security.json @@ -23,6 +23,9 @@ "requireReleaseEvidence": true }, "releaseEvidence": { + "evidenceSanitizer": "scripts/sanitize-evidence.sh", + "artifactArchivePolicy": "hashes-only-for-reproducibility", + "forbiddenEvidenceMaterial": ["host-private-paths", "raw-git-remotes", "private-keys", "credential-shaped-material"], "targetSoakProof": { "requiredForProduction": true, "proofFileEnv": "JERBOA_DB_TARGET_SOAK_PROOF_FILE", --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ JERBOA_VERSION ?= v0.2.3 JERBOA_TOOL_DIR ?= $(CURDIR)/.jerboa/bin -JERBUILD ?= $(if $(wildcard $(JERBOA_TOOL_DIR)/jerbuild),$(JERBOA_TOOL_DIR)/jerbuild,jerbuild) +ADJACENT_JERBUILD := $(CURDIR)/../jerboa/dist/jerbuild +JERBUILD ?= $(if $(wildcard $(ADJACENT_JERBUILD)),$(ADJACENT_JERBUILD),$(if $(wildcard $(JERBOA_TOOL_DIR)/jerbuild),$(JERBOA_TOOL_DIR)/jerbuild,jerbuild)) JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) ifeq ($(JH),) JERBOA_DIR ?= $(realpath $(CURDIR)/../jerboa) @@ -148,15 +149,21 @@ release-evidence: verify reproducibility-report sbom soak-evidence mkdir -p "$(DIST_DIR)" git rev-parse HEAD > "$(DIST_DIR)/git-commit.txt" git status --short > "$(DIST_DIR)/git-status.txt" - uname -a > "$(DIST_DIR)/build-environment.txt" - @printf 'JERBOA_DIR=%s\nSCHEME=%s\nJERBOA_NATIVE_TARGET_DIR=%s\nJERBOA_NATIVE_FEATURES=%s\nJERBOA_NATIVE_NO_DEFAULT_FEATURES=%s\nJERBOA_NATIVE_LIB=%s\n' "$(JERBOA_DIR)" "$(SCHEME)" "$(JERBOA_NATIVE_TARGET_DIR)" "$(JERBOA_NATIVE_FEATURES)" "$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)" "$(JERBOA_NATIVE_LIB)" >> "$(DIST_DIR)/build-environment.txt" + @{ \ + printf 'os=%s\n' "$$(uname -s 2>/dev/null || printf unknown)"; \ + printf 'kernel_release=%s\n' "$$(uname -r 2>/dev/null || printf unknown)"; \ + printf 'machine=%s\n' "$$(uname -m 2>/dev/null || printf unknown)"; \ + if [ -d "$(JERBOA_DIR)" ]; then printf 'jerboa_dir_status=present\n'; else printf 'jerboa_dir_status=missing\n'; fi; \ + if [ -x "$(SCHEME)" ]; then printf 'scheme_status=present\n'; else printf 'scheme_status=unavailable-cache-uses-jerbuild-exec\n'; fi; \ + case "$(JERBOA_NATIVE_TARGET_DIR)" in "$(CURDIR)"/.jerboa/*) printf 'jerboa_native_target_dir_status=repo-local\n' ;; *) printf 'jerboa_native_target_dir_status=external\n' ;; esac; \ + printf 'JERBOA_NATIVE_FEATURES=%s\n' "$(JERBOA_NATIVE_FEATURES)"; \ + printf 'JERBOA_NATIVE_NO_DEFAULT_FEATURES=%s\n' "$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)"; \ + if [ -e "$(JERBOA_NATIVE_LIB)" ]; then printf 'jerboa_native_lib_status=present\n'; else printf 'jerboa_native_lib_status=missing\n'; fi; \ + } > "$(DIST_DIR)/build-environment.txt" @if [ -x "$(SCHEME)" ]; then \ "$(SCHEME)" --version >> "$(DIST_DIR)/build-environment.txt" 2>&1; \ - else \ - printf 'SCHEME_STATUS=unavailable-cache-uses-jerbuild-exec\n' >> "$(DIST_DIR)/build-environment.txt"; \ fi @$(JERBUILD) --version >> "$(DIST_DIR)/build-environment.txt" 2>&1 || true - @$(JERBUILD) --jerboa-home >> "$(DIST_DIR)/build-environment.txt" 2>&1 || true @$(MAKE) security > "$(DIST_DIR)/security-check.txt" @$(MAKE) build > "$(DIST_DIR)/import-check.txt" @$(MAKE) test > "$(DIST_DIR)/test.txt" @@ -165,10 +172,11 @@ release-evidence: verify reproducibility-report sbom soak-evidence cargo metadata --manifest-path "$(JERBOA_NATIVE_MANIFEST)" --locked --format-version 1 > "$(DIST_DIR)/cargo-metadata-jerboa-native.json" 2>/dev/null || true; \ fi find lib bin tests benchmarks examples support -type f -name '*.ss' -print | sort | xargs shasum -a 256 > "$(DIST_DIR)/source-sha256.txt" - shasum -a 256 Makefile .jerboa/security.json SECURITY.md README.md docs/threat-model.md docs/storage-hardening.md docs/release-evidence.md scripts/security-check.sh scripts/sbom.sh scripts/reproducibility-report.sh scripts/soak-evidence.sh support/import-check.ss support/run-ss.sh > "$(DIST_DIR)/release-inputs-sha256.txt" + shasum -a 256 Makefile .jerboa/security.json SECURITY.md README.md docs/threat-model.md docs/storage-hardening.md docs/release-evidence.md scripts/security-check.sh scripts/sbom.sh scripts/reproducibility-report.sh scripts/soak-evidence.sh scripts/sanitize-evidence.sh support/import-check.ss support/run-ss.sh > "$(DIST_DIR)/release-inputs-sha256.txt" cp -R "$(SBOM_DIR)" "$(DIST_DIR)/sbom" cp -R "$(REPRO_DIR)" "$(DIST_DIR)/reproducibility" cp -R "$(SOAK_DIR)" "$(DIST_DIR)/soak" + scripts/sanitize-evidence.sh "$(DIST_DIR)" "$(SBOM_DIR)" "$(REPRO_DIR)" "$(SOAK_DIR)" showcase: ensure-jerboa-tools $(RUN_SS) examples/bookstore.ss --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -20,7 +20,8 @@ It records: - `sbom/` with source, release-input, sibling-native metadata, and no-project-local-package-manager dependency provenance. - `reproducibility/` with two repeated local security/import/test passes and - `result.txt`. + `result.txt`. The reproducibility archive is hash-only; scratch build logs + and raw gate output are not preserved. - `soak/` with local cluster smoke status, bounded TCP transport smoke status, authenticated frame smoke status, DuckDB/LevelDB backend smoke status, and explicit blocked status for target-host peer HTTP and sustained load gates. @@ -43,6 +44,14 @@ Required release markers: - `target_peer_http_proof_status: documented` - `leveldb_proof_status: documented` - `proof_failure_status: blocked-target-proof` +- `evidence_sanitizer_status: documented` +- `reproducibility_archive_status: hashes-only` + +Evidence is sanitized before publication. Local host paths, private Git remote +syntax, and machine-specific hostnames are replaced with neutral markers. +Target proof files are rejected before copy when they are larger than +`JERBOA_DB_TARGET_PROOF_MAX_BYTES` bytes or contain host-private paths, raw +`git@` remotes, private-key blocks, or credential-shaped material. `make verify` is the deterministic local gate. The following production gates are tracked but not part of the default Darwin-local gate: @@ -69,6 +78,9 @@ Release CI can require target-host evidence by setting the matching `JERBOA_DB_REQUIRE_*_PROOF=1` variable. Missing or marker-incomplete proof files fail `make soak-evidence` with `status=blocked-target-proof`; valid proofs are copied into `dist/release-evidence/soak/` with `.sha256` sidecars. +Oversized proof files fail as `too-large`, host-private proof files fail as +`private-material`, and key/token-shaped proof files fail as +`sensitive-material`. - `JERBOA_DB_TARGET_SOAK_PROOF_FILE` with `JERBOA_DB_REQUIRE_TARGET_SOAK_PROOF=1` records sustained release-host --- a/scripts/reproducibility-report.sh +++ b/scripts/reproducibility-report.sh @@ -8,6 +8,12 @@ OUT=${JERBOA_DB_REPRO_DIR:-dist/reproducibility} WORK=$(mktemp -d "${TMPDIR:-/tmp}/jerboa-db-repro.XXXXXX") trap 'rm -rf "$WORK"' EXIT HUP INT TERM +hash_file_as() { + file=$1 + label=$2 + shasum -a 256 "$file" | awk -v label="$label" '{ print $1 " " label }' +} + run_pass() { name=$1 dir="$WORK/$name" @@ -31,6 +37,7 @@ run_pass() { scripts/sbom.sh \ scripts/reproducibility-report.sh \ scripts/soak-evidence.sh \ + scripts/sanitize-evidence.sh \ support/import-check.ss \ support/run-ss.sh > "$dir/release-inputs-sha256.txt" } @@ -51,14 +58,22 @@ run_pass second rm -rf "$OUT" mkdir -p "$OUT" -cp -R "$WORK/first" "$OUT/first" -cp -R "$WORK/second" "$OUT/second" +cp "$WORK/first/source-sha256.txt" "$OUT/first-source.sha256" +cp "$WORK/second/source-sha256.txt" "$OUT/second-source.sha256" +cp "$WORK/first/release-inputs-sha256.txt" "$OUT/first-release-inputs.sha256" +cp "$WORK/second/release-inputs-sha256.txt" "$OUT/second-release-inputs.sha256" +hash_file_as "$WORK/first/security.txt" first-security-output > "$OUT/first-security-output.sha256" +hash_file_as "$WORK/second/security.txt" second-security-output > "$OUT/second-security-output.sha256" +hash_file_as "$WORK/first/import-check.txt" first-import-output > "$OUT/first-import-output.sha256" +hash_file_as "$WORK/second/import-check.txt" second-import-output > "$OUT/second-import-output.sha256" +hash_file_as "$WORK/first/test.txt" first-test-output > "$OUT/first-test-output.sha256" +hash_file_as "$WORK/second/test.txt" second-test-output > "$OUT/second-test-output.sha256" -source_status=$(cmp_status "$OUT/first/source-sha256.txt" "$OUT/second/source-sha256.txt") -release_inputs_status=$(cmp_status "$OUT/first/release-inputs-sha256.txt" "$OUT/second/release-inputs-sha256.txt") -security_output_status=$(cmp_status "$OUT/first/security.txt" "$OUT/second/security.txt") -import_output_status=$(cmp_status "$OUT/first/import-check.txt" "$OUT/second/import-check.txt") -test_output_status=$(cmp_status "$OUT/first/test.txt" "$OUT/second/test.txt") +source_status=$(cmp_status "$WORK/first/source-sha256.txt" "$WORK/second/source-sha256.txt") +release_inputs_status=$(cmp_status "$WORK/first/release-inputs-sha256.txt" "$WORK/second/release-inputs-sha256.txt") +security_output_status=$(cmp_status "$WORK/first/security.txt" "$WORK/second/security.txt") +import_output_status=$(cmp_status "$WORK/first/import-check.txt" "$WORK/second/import-check.txt") +test_output_status=$(cmp_status "$WORK/first/test.txt" "$WORK/second/test.txt") status=match for value in "$source_status" "$release_inputs_status" "$security_output_status" "$import_output_status" "$test_output_status"; do @@ -69,6 +84,8 @@ done { echo "repo=jerboa-db" + echo "artifact_archive_status=hashes-only" + echo "scratch_build_logs_status=not-archived" echo "source_status=$source_status" echo "release_inputs_status=$release_inputs_status" echo "security_output_status=$security_output_status" new file mode 100755 --- /dev/null +++ b/scripts/sanitize-evidence.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +if [ "$#" -eq 0 ]; then + set -- dist/release-evidence +fi + +for evidence_dir in "$@"; do + if [ ! -d "$evidence_dir" ]; then + continue + fi + + find "$evidence_dir" -type f -print | while IFS= read -r file; do + perl -0pi -e ' + s#/Users/[^[:space:]\x22\x27()]+#<local-path>#g; + if (defined $ENV{HOME} && length $ENV{HOME}) { + my $home = quotemeta($ENV{HOME}); + s#$home#<home>#g; + } + s#~/mine#<local-path>#g; + s#\$\((HOME)\)/mine#<local-path>#g; + s#users-MacBook-Pro#<host>#g; + s#git\@#git-at-#g; + ' "$file" + done +done --- a/scripts/sbom.sh +++ b/scripts/sbom.sh @@ -11,9 +11,13 @@ JERBOA_NATIVE_TARGET_DIR=${JERBOA_NATIVE_TARGET_DIR:-} JERBOA_NATIVE_FEATURES=${JERBOA_NATIVE_FEATURES:-} JERBOA_NATIVE_NO_DEFAULT_FEATURES=${JERBOA_NATIVE_NO_DEFAULT_FEATURES:-} JERBOA_NATIVE_LIB=${JERBOA_NATIVE_LIB:-} +SANITIZER="$ROOT/scripts/sanitize-evidence.sh" rm -rf "$OUT" mkdir -p "$OUT" +if [ -x "$SANITIZER" ]; then + trap '"$SANITIZER" "$OUT"' EXIT +fi { echo "repo=jerboa-db" @@ -23,25 +27,30 @@ mkdir -p "$OUT" if [ -x "$JERBUILD" ] || command -v "$JERBUILD" >/dev/null 2>&1; then printf 'jerbuild_version=' "$JERBUILD" --version 2>/dev/null || true - printf 'jerboa_home=' - "$JERBUILD" --jerboa-home 2>/dev/null || true + if "$JERBUILD" --jerboa-home >/dev/null 2>&1; then + echo "jerboa_home_status=present" + else + echo "jerboa_home_status=missing" + fi fi echo "project_local_package_manager_dependencies=none" if [ -n "$JERBOA_NATIVE_MANIFEST" ] && [ -f "$JERBOA_NATIVE_MANIFEST" ]; then - echo "sibling_native_manifest=$JERBOA_NATIVE_MANIFEST" + echo "sibling_native_manifest_status=present" echo "sibling_native_metadata_status=present" else - echo "sibling_native_manifest=missing" + echo "sibling_native_manifest_status=missing" echo "sibling_native_metadata_status=missing" fi - echo "jerboa_native_target_dir=$JERBOA_NATIVE_TARGET_DIR" + case "$JERBOA_NATIVE_TARGET_DIR" in + "$ROOT"/.jerboa/*) echo "jerboa_native_target_dir_status=repo-local" ;; + '') echo "jerboa_native_target_dir_status=missing" ;; + *) echo "jerboa_native_target_dir_status=external" ;; + esac echo "jerboa_native_features=$JERBOA_NATIVE_FEATURES" echo "jerboa_native_no_default_features=$JERBOA_NATIVE_NO_DEFAULT_FEATURES" if [ -n "$JERBOA_NATIVE_LIB" ] && [ -f "$JERBOA_NATIVE_LIB" ]; then - echo "jerboa_native_lib=$JERBOA_NATIVE_LIB" echo "jerboa_native_lib_status=present" else - echo "jerboa_native_lib=missing" echo "jerboa_native_lib_status=missing" fi } > "$OUT/manifest.txt" @@ -78,5 +87,6 @@ if [ -n "$JERBOA_NATIVE_MANIFEST" ] && [ -f "$JERBOA_NATIVE_MANIFEST" ]; then fi if [ -n "$JERBOA_NATIVE_LIB" ] && [ -f "$JERBOA_NATIVE_LIB" ]; then - shasum -a 256 "$JERBOA_NATIVE_LIB" > "$OUT/jerboa-native-lib-sha256.txt" + shasum -a 256 "$JERBOA_NATIVE_LIB" | + awk '{ print $1 " native/libjerboa_native" }' > "$OUT/jerboa-native-lib-sha256.txt" fi --- a/scripts/security-check.sh +++ b/scripts/security-check.sh @@ -25,6 +25,7 @@ require_file support/run-ss.sh require_file scripts/sbom.sh require_file scripts/reproducibility-report.sh require_file scripts/soak-evidence.sh +require_file scripts/sanitize-evidence.sh for marker in \ 'transport_timeout_status: documented' \ @@ -39,7 +40,9 @@ for marker in \ 'target_soak_proof_status: documented' \ 'target_peer_http_proof_status: documented' \ 'leveldb_proof_status: documented' \ - 'proof_failure_status: blocked-target-proof' + 'proof_failure_status: blocked-target-proof' \ + 'evidence_sanitizer_status: documented' \ + 'reproducibility_archive_status: hashes-only' do if ! grep -q "$marker" docs/release-evidence.md docs/storage-hardening.md; then note_fail "missing soak/transport marker: $marker" @@ -53,6 +56,10 @@ for marker in \ 'JERBOA_DB_REQUIRE_TARGET_PEER_HTTP_PROOF' \ 'JERBOA_DB_LEVELDB_PROOF_FILE' \ 'JERBOA_DB_REQUIRE_LEVELDB_PROOF' \ + 'JERBOA_DB_TARGET_PROOF_MAX_BYTES' \ + 'private-material' \ + 'sensitive-material' \ + 'too-large' \ 'blocked-target-proof' do if ! grep -q "$marker" scripts/soak-evidence.sh; then @@ -60,6 +67,14 @@ do fi done +[ -x scripts/sanitize-evidence.sh ] || note_fail "scripts/sanitize-evidence.sh must be executable" +grep -q 'sanitize-evidence.sh' Makefile || note_fail "release-evidence must run the evidence sanitizer" +grep -q 'artifact_archive_status=hashes-only' scripts/reproducibility-report.sh || note_fail "reproducibility evidence must be hash-only" +grep -q 'scratch_build_logs_status=not-archived' scripts/reproducibility-report.sh || note_fail "scratch build logs must not be archived" +if grep -R -n 'uname -a' Makefile scripts docs SECURITY.md .jerboa/security.json | grep -v '^scripts/security-check.sh:'; then + note_fail "raw uname -a must not be recorded in release evidence" +fi + if grep -R -n -I -E '^\(import[[:space:]]+\(chezscheme\)' bin tests benchmarks examples lib 2>/dev/null; then note_fail "direct top-level (chezscheme) import remains in runnable source" fi --- a/scripts/soak-evidence.sh +++ b/scripts/soak-evidence.sh @@ -18,9 +18,25 @@ TARGET_PEER_HTTP_PROOF=${JERBOA_DB_TARGET_PEER_HTTP_PROOF_FILE:-} REQUIRE_TARGET_PEER_HTTP_PROOF=${JERBOA_DB_REQUIRE_TARGET_PEER_HTTP_PROOF:-0} LEVELDB_PROOF=${JERBOA_DB_LEVELDB_PROOF_FILE:-} REQUIRE_LEVELDB_PROOF=${JERBOA_DB_REQUIRE_LEVELDB_PROOF:-0} +MAX_PROOF_BYTES=${JERBOA_DB_TARGET_PROOF_MAX_BYTES:-65536} +SANITIZER="$ROOT/scripts/sanitize-evidence.sh" rm -rf "$OUT" mkdir -p "$OUT" +if [ -x "$SANITIZER" ]; then + trap '"$SANITIZER" "$OUT"' EXIT +fi + +case "$MAX_PROOF_BYTES" in + ''|*[!0-9]*) + echo "JERBOA_DB_TARGET_PROOF_MAX_BYTES must be a positive integer" >&2 + exit 1 + ;; +esac +if [ "$MAX_PROOF_BYTES" -lt 1 ]; then + echo "JERBOA_DB_TARGET_PROOF_MAX_BYTES must be a positive integer" >&2 + exit 1 +fi target_soak_proof_status=not-run target_peer_http_proof_status=not-run @@ -43,6 +59,37 @@ require_marker() { fi } +proof_contains() { + pattern=$1 + file=$2 + if command -v rg >/dev/null 2>&1; then + rg -q -I -e "$pattern" "$file" 2>/dev/null + else + grep -E -q "$pattern" "$file" 2>/dev/null + fi +} + +validate_proof_material() { + proof=$1 + proof_name=$2 + size=$(wc -c < "$proof" | tr -d ' ') + if [ "$size" -gt "$MAX_PROOF_BYTES" ]; then + write_blocked_target_proof "$proof_name" too-large "target proof is larger than JERBOA_DB_TARGET_PROOF_MAX_BYTES" + return 1 + fi + + secret_pattern='(BEGIN (RSA|OPENSSH|EC|DSA|PRIVATE) KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16}|db[_-]?(secret|token|password)[[:space:]]*[:=]|transport[_-]?secret[[:space:]]*[:=])' + private_pattern='(/Users/|~/mine|\$\(HOME\)/mine|git@|users-MacBook-Pro|uname[[:space:]]+-a)' + if proof_contains "$secret_pattern" "$proof"; then + write_blocked_target_proof "$proof_name" sensitive-material "target proof contains key/token-shaped material" + return 1 + fi + if proof_contains "$private_pattern" "$proof"; then + write_blocked_target_proof "$proof_name" private-material "target proof contains host-private material" + return 1 + fi +} + record_proof() { proof=$1 destination=$2 @@ -94,6 +141,7 @@ if [ "$target_soak_proof_requested" = "1" ]; then write_blocked_target_proof target-soak missing "set JERBOA_DB_TARGET_SOAK_PROOF_FILE to a non-empty release-host soak proof" exit 1 fi + validate_proof_material "$TARGET_SOAK_PROOF" target-soak || exit 1 if ! require_marker "$TARGET_SOAK_PROOF" "production_soak_status=current-run-recorded" "target soak" || ! require_marker "$TARGET_SOAK_PROOF" "cluster_status=release-host-sustained-recorded" "target soak" || ! require_marker "$TARGET_SOAK_PROOF" "transport_status=release-host-sustained-recorded" "target soak" || @@ -116,6 +164,7 @@ if [ "$target_peer_http_proof_requested" = "1" ]; then write_blocked_target_proof target-peer-http missing "set JERBOA_DB_TARGET_PEER_HTTP_PROOF_FILE to a non-empty Linux peer HTTP proof" exit 1 fi + validate_proof_material "$TARGET_PEER_HTTP_PROOF" target-peer-http || exit 1 if ! require_marker "$TARGET_PEER_HTTP_PROOF" "peer_http_status=linux-release-host-recorded" "target peer HTTP" || ! require_marker "$TARGET_PEER_HTTP_PROOF" "epoll_native_status=linux-epoll-recorded" "target peer HTTP" || ! require_marker "$TARGET_PEER_HTTP_PROOF" "peer_http_auth_status=target-evidence-recorded" "target peer HTTP"; then @@ -136,6 +185,7 @@ if [ "$leveldb_proof_requested" = "1" ]; then write_blocked_target_proof leveldb missing "set JERBOA_DB_LEVELDB_PROOF_FILE to a non-empty LevelDB backend proof" exit 1 fi + validate_proof_material "$LEVELDB_PROOF" leveldb || exit 1 if ! require_marker "$LEVELDB_PROOF" "leveldb_status=target-evidence-recorded" "LevelDB" || ! require_marker "$LEVELDB_PROOF" "chez_leveldb_status=pinned-version-recorded" "LevelDB" || ! require_marker "$LEVELDB_PROOF" "leveldb_native_status=pinned-version-recorded" "LevelDB" || @@ -271,6 +321,7 @@ fi echo "leveldb_status=$leveldb_status" echo "production_soak_status=$production_soak_status" echo "production_min_seconds=$PRODUCTION_MIN_SECONDS" + echo "target_proof_max_bytes=$MAX_PROOF_BYTES" echo "target_soak_proof_status=$target_soak_proof_status" echo "target_peer_http_proof_status=$target_peer_http_proof_status" echo "leveldb_proof_status=$leveldb_proof_status"