Harden HTTPD release evidence privacy
ober
4154bb35dc1cf6a566b19468f96eded95a62c3c4
--- a/.jerboa/security.json +++ b/.jerboa/security.json @@ -43,7 +43,20 @@ "forwarded_header_policy_status=deny-by-default", "plaintext_listener_policy_status=private-or-behind-tls-proxy" ] - } + }, + "evidenceSanitizer": "scripts/sanitize-evidence.sh", + "artifactArchivePolicy": "hashes-only", + "forbiddenEvidenceMaterial": [ + "private paths", + "SSH clone URLs", + "full uname output", + "operational hostnames", + "TLS private keys", + "authorization headers", + "cookies", + "tokens", + "passwords" + ] } }, "suppressions": [] --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -JERBUILD ?= jerbuild +ADJACENT_JERBUILD := $(CURDIR)/../jerboa/dist/jerbuild +JERBUILD ?= $(if $(wildcard $(ADJACENT_JERBUILD)),$(ADJACENT_JERBUILD),jerbuild) JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) ifeq ($(JH),) $(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) @@ -105,12 +106,22 @@ tls-proxy-evidence: verify: security test fuzz-check load-smoke audit sbom reproducibility-report release-evidence: verify + rm -rf "$(DIST_DIR)" mkdir -p "$(DIST_DIR)" git rev-parse HEAD > "$(DIST_DIR)/git-commit.txt" git status --short > "$(DIST_DIR)/git-status.txt" - uname -a > "$(DIST_DIR)/build-environment.txt" - $(JERBUILD) --version >> "$(DIST_DIR)/build-environment.txt" - $(JERBUILD) --jerboa-home >> "$(DIST_DIR)/build-environment.txt" + { \ + printf 'os=%s\n' "$$(uname -s)"; \ + printf 'kernel_release=%s\n' "$$(uname -r)"; \ + printf 'machine=%s\n' "$$(uname -m)"; \ + printf 'jerbuild_version=\n'; \ + $(JERBUILD) --version; \ + if [ -n "$(JH)" ]; then \ + printf 'jerboa_home_status=present\n'; \ + else \ + printf 'jerboa_home_status=missing\n'; \ + fi; \ + } > "$(DIST_DIR)/build-environment.txt" cargo metadata --manifest-path wasm/Cargo.toml --locked --format-version 1 > "$(DIST_DIR)/cargo-metadata-wasm.json" cargo metadata --manifest-path fuzz/Cargo.toml --locked --format-version 1 > "$(DIST_DIR)/cargo-metadata-fuzz.json" find lib/jerboa-https/sandbox -type f -name '*.wasm' -exec shasum -a 256 {} \; > "$(DIST_DIR)/wasm-sha256.txt" @@ -129,12 +140,20 @@ release-evidence: verify cp -R "$(TLS_PROXY_EVIDENCE_DIR)" "$(DIST_DIR)/tls-proxy-evidence" cat "$(SBOM_DIR)/status.txt" > "$(DIST_DIR)/sbom.log" cat "$(REPRO_DIR)/result.txt" > "$(DIST_DIR)/reproducibility.log" + grep -q '^artifact_archive_status=hashes-only$$' "$(REPRO_DIR)/result.txt" + grep -q '^scratch_build_logs_status=not-archived$$' "$(REPRO_DIR)/result.txt" cat "$(SOAK_DIR)/status.txt" > "$(DIST_DIR)/soak.log" cat "$(FUZZ_EVIDENCE_DIR)/status.txt" > "$(DIST_DIR)/fuzz-evidence.log" cat "$(TLS_PROXY_EVIDENCE_DIR)/status.txt" > "$(DIST_DIR)/tls-proxy-policy.txt" grep -q '^tls_proxy_policy_status=present$$' "$(DIST_DIR)/tls-proxy-policy.txt" grep -q '^forwarded_header_policy_status=deny-by-default$$' "$(DIST_DIR)/tls-proxy-policy.txt" grep -q '^plaintext_listener_policy_status=private-or-behind-tls-proxy$$' "$(DIST_DIR)/tls-proxy-policy.txt" + scripts/sanitize-evidence.sh "$(DIST_DIR)" + scripts/sanitize-evidence.sh "$(SBOM_DIR)" + scripts/sanitize-evidence.sh "$(REPRO_DIR)" + scripts/sanitize-evidence.sh "$(SOAK_DIR)" + scripts/sanitize-evidence.sh "$(FUZZ_EVIDENCE_DIR)" + scripts/sanitize-evidence.sh "$(TLS_PROXY_EVIDENCE_DIR)" if [ "$${JHTTPS_RUN_COVERAGE_FUZZ:-0}" = "1" ]; then grep -q '^coverage_fuzz_status=local-smoke-recorded$$' "$(DIST_DIR)/fuzz-evidence/status.txt"; fi clean: --- a/SECURITY.md +++ b/SECURITY.md @@ -33,8 +33,14 @@ test in this checkout, but required proof must fail closed. Use `JHTTPS_REQUIRE_TARGET_LOAD_PROOF=1` with `JHTTPS_TARGET_LOAD_PROOF_FILE`, `JHTTPS_REQUIRE_TARGET_FUZZ_PROOF=1` with `JHTTPS_TARGET_FUZZ_PROOF_FILE`, and `JHTTPS_REQUIRE_TARGET_TLS_PROXY_PROOF=1` with -`JHTTPS_TARGET_TLS_PROXY_PROOF_FILE`. Missing, empty, or marker-incomplete proof -records `status=blocked-target-proof` and does not clear production blockers. +`JHTTPS_TARGET_TLS_PROXY_PROOF_FILE`. Missing, empty, overlarge, private, +sensitive, or marker-incomplete proof records `status=blocked-target-proof` +and does not clear production blockers. + +Release evidence must be safe to publish: generated evidence is sanitized for +private paths, SSH clone URLs, full host identity, and hostnames; +reproducibility evidence archives only hashes/diffs/status, not scratch +generated libraries or build logs. ## Hardening Expectations --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -19,8 +19,9 @@ artifacts under `dist/release-evidence/`: - `build-environment.txt` with OS and Jerboa toolchain identity. - `sbom/` and `sbom.log` with toolchain identity, source hashes, generated library hashes, and local `jerboa-ssl` dependency hashes. -- `reproducibility/` and `reproducibility.log` with two clean generated-library - builds, SHA-256 manifests, and any manifest/tree diffs. +- `reproducibility/` and `reproducibility.log` with SHA-256 manifests, diffs, + and status for two clean generated-library builds. Raw generated libraries + and scratch build logs are not archived in release evidence. - `soak/` and `soak.log` with HTTPD slow-client/load/queue-exhaustion evidence status. The default local release bundle records `blocked-not-run`. Opt-in local evidence records deterministic slow-client coverage plus bounded @@ -87,6 +88,12 @@ load_status=release-host-sustained-recorded queue_exhaustion_status=release-host-sustained-recorded ``` +Supplied load proof is size-limited, checked for private paths and +high-confidence sensitive material, copied into `dist/soak/`, and hashed only +after those checks pass. Missing, empty, overlarge, private, sensitive, or +marker-mismatched required proof fails closed with +`status=blocked-target-proof`. + To refresh only the HTTPD fuzz evidence, run: ```sh @@ -117,6 +124,10 @@ coverage_fuzz_status=release-host-sustained-recorded targets_completed=3 ``` +Supplied fuzz proof is size-limited, checked for private paths and +high-confidence sensitive material, copied into `dist/fuzz-evidence/`, and +hashed only after those checks pass. + To attach reviewed target-host TLS/proxy smoke proof, provide a text proof file and require it: @@ -133,3 +144,7 @@ tls_proxy_smoke_status=target-evidence-recorded forwarded_header_policy_status=deny-by-default plaintext_listener_policy_status=private-or-behind-tls-proxy ``` + +Supplied TLS/proxy proof is size-limited, checked for private paths and +high-confidence sensitive material, copied into `dist/tls-proxy-evidence/`, +and hashed only after those checks pass. --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -1,8 +1,8 @@ # jerboa-https HTTPD Threat Model Status: the HTTP client library is a review-required dependency; the HTTP/1.1 -server module is experimental/private until the daemon release gates in -`~/Release-plan.md` and `~/mine/jerboa-production-readiness.md` are complete. +server module is experimental/private until the daemon release gates in the +release plan and production-readiness tracker are complete. ## Scope --- a/scripts/daemon-security-check.sh +++ b/scripts/daemon-security-check.sh @@ -43,6 +43,7 @@ require_file "scripts/reproducibility-report.sh" require_file "scripts/soak-evidence.sh" require_file "scripts/fuzz-evidence.sh" require_file "scripts/tls-proxy-evidence.sh" +require_file "scripts/sanitize-evidence.sh" require_file "wasm/Cargo.toml" require_file "wasm/http/Cargo.toml" require_file "wasm/http/src/lib.rs" @@ -90,6 +91,23 @@ require_text 'blocked-target-proof' scripts/fuzz-evidence.sh require_text 'JHTTPS_TARGET_TLS_PROXY_PROOF_FILE' scripts/tls-proxy-evidence.sh require_text 'JHTTPS_REQUIRE_TARGET_TLS_PROXY_PROOF' scripts/tls-proxy-evidence.sh require_text 'blocked-target-proof' scripts/tls-proxy-evidence.sh +require_text 'scripts/sanitize-evidence.sh' Makefile +require_text 'artifact_archive_status=hashes-only' scripts/reproducibility-report.sh +require_text 'scratch_build_logs_status=not-archived' scripts/reproducibility-report.sh + +if command -v rg >/dev/null 2>&1; then + full_uname_pattern='uname -' + full_uname_pattern="${full_uname_pattern}a" + if rg -n -S -F "${rg_excludes[@]}" -g 'Makefile' -g 'scripts/*.sh' -g 'docs/*.md' "$full_uname_pattern" .; then + printf '[daemon-security] release evidence must not capture full uname output\n' >&2 + fail=1 + fi +fi + +if [ ! -x scripts/sanitize-evidence.sh ]; then + printf '[daemon-security] scripts/sanitize-evidence.sh must be executable\n' >&2 + fail=1 +fi if command -v rg >/dev/null 2>&1; then say "running high-confidence secret scan" @@ -134,7 +152,7 @@ elif command -v gitleaks >/dev/null 2>&1; then fail=1 fi else - warn "gitsafe not found; install ~/mine/jerboa-gitsafe for release-grade secret scanning" + warn "gitsafe not found; install jerboa-gitsafe for release-grade secret scanning" fi exit "$fail" --- a/scripts/fuzz-evidence.sh +++ b/scripts/fuzz-evidence.sh @@ -22,14 +22,28 @@ require_target_fuzz_proof="${JHTTPS_REQUIRE_TARGET_FUZZ_PROOF:-0}" rm -rf "$out_dir" mkdir -p "$out_dir/artifacts" "$out_dir/work-corpus" +sanitize_output() { + if [ -x "$repo_root/scripts/sanitize-evidence.sh" ]; then + "$repo_root/scripts/sanitize-evidence.sh" "$out_dir" + fi +} +trap sanitize_output EXIT + proof_status_file="$out_dir/proof-status.txt" : > "$proof_status_file" proof_blocked=0 +proof_max_bytes="${JHTTPS_TARGET_PROOF_MAX_BYTES:-65536}" + +proof_contains() { + local pattern=$1 + local file=$2 + LC_ALL=C grep -E -q "$pattern" "$file" +} copy_proof() { local source_path=$1 local required=$2 - local proof_path marker + local proof_path marker bytes secret_pattern private_pattern if [ -z "$source_path" ]; then if [ "$required" = "1" ]; then @@ -52,6 +66,35 @@ copy_proof() { return 0 fi + bytes="$(wc -c < "$proof_path" | tr -d '[:space:]')" + case "$bytes" in + ''|*[!0-9]*) + printf 'target_fuzz_proof_status=blocked-invalid-size\n' >> "$proof_status_file" + proof_blocked=1 + return 0 + ;; + *) + if [ "$bytes" -gt "$proof_max_bytes" ]; then + printf 'target_fuzz_proof_status=blocked-overlarge\n' >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + ;; + esac + + secret_pattern='(BEGIN [A-Z ]*PRIVATE KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16}|tls[_-]?private[_-]?key[[:space:]]*[:=]|auth[_-]?secret[[:space:]]*[:=]|password[[:space:]]*[:=]|token[[:space:]]*[:=])' + private_pattern='(/Users/|~/mine|\$\(HOME\)/mine|git@|users-MacBook-Pro|uname[[:space:]]+-a)' + if proof_contains "$secret_pattern" "$proof_path"; then + printf 'target_fuzz_proof_status=blocked-sensitive-material\n' >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + if proof_contains "$private_pattern" "$proof_path"; then + printf 'target_fuzz_proof_status=blocked-private-material\n' >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + for marker in \ '^coverage_fuzz_status=release-host-sustained-recorded$' \ '^targets_completed=3$' @@ -86,19 +129,45 @@ fi { printf 'generated_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - printf 'jerbuild_command=%s\n' "$jerbuild_cmd" - printf 'ssl_dir=%s\n' "$ssl_dir" + if [ -n "$jerbuild_cmd" ]; then + printf 'jerbuild_status=selected\n' + else + printf 'jerbuild_status=missing\n' + fi + if [ -d "$ssl_dir" ]; then + printf 'ssl_dir_status=present\n' + else + printf 'ssl_dir_status=missing\n' + fi printf 'deterministic_requested=%s\n' "$run_deterministic" printf 'coverage_guided_requested=%s\n' "$run_coverage" printf 'runs_per_target=%s\n' "$runs" printf 'targets=%s\n' "$targets" - printf 'target_fuzz_proof_file=%s\n' "$target_fuzz_proof_file" + if [ -n "$target_fuzz_proof_file" ]; then + printf 'target_fuzz_proof_file_status=provided\n' + else + printf 'target_fuzz_proof_file_status=not-provided\n' + fi printf 'deterministic_corpus=tests/httpd-fuzz-test.ss\n' printf 'coverage_guided_harness=fuzz/fuzz_targets\n' - printf 'uname=%s\n' "$(uname -a)" - printf 'cargo=%s\n' "$(command -v cargo || true)" - printf 'cargo-fuzz=%s\n' "$(command -v cargo-fuzz || true)" - printf 'nightly_cargo=%s\n' "$(rustup which cargo --toolchain nightly 2>/dev/null || true)" + printf 'os=%s\n' "$(uname -s)" + printf 'kernel_release=%s\n' "$(uname -r)" + printf 'machine=%s\n' "$(uname -m)" + if command -v cargo >/dev/null 2>&1; then + printf 'cargo_status=present\n' + else + printf 'cargo_status=missing\n' + fi + if command -v cargo-fuzz >/dev/null 2>&1 || [ -x "$HOME/.cargo/bin/cargo-fuzz" ]; then + printf 'cargo_fuzz_status=present\n' + else + printf 'cargo_fuzz_status=missing\n' + fi + if rustup which cargo --toolchain nightly >/dev/null 2>&1; then + printf 'nightly_cargo_status=present\n' + else + printf 'nightly_cargo_status=missing\n' + fi } > "$out_dir/environment.txt" if [ "$proof_blocked" -ne 0 ]; then --- a/scripts/reproducibility-report.sh +++ b/scripts/reproducibility-report.sh @@ -12,57 +12,93 @@ make_cmd="${MAKE:-make}" jerbuild_cmd="${JERBUILD:-jerbuild}" ssl_dir="${SSL_DIR:-$repo_root/../jerboa-ssl}" require_match="${REQUIRE_REPRODUCIBLE:-0}" +tmp_parent="${TMPDIR:-/tmp}" +tmp_dir="$(mktemp -d "${tmp_parent%/}/jhttps-repro-XXXXXX")" + +cleanup() { + if [ -x "$repo_root/scripts/sanitize-evidence.sh" ]; then + "$repo_root/scripts/sanitize-evidence.sh" "$out_dir" + fi + rm -rf "$tmp_dir" +} +trap cleanup EXIT run_build() { local label="$1" local dest="$2" - "$make_cmd" -C "$repo_root" clean > "$out_dir/$label-clean.log" 2>&1 - "$make_cmd" -C "$repo_root" build "JERBUILD=$jerbuild_cmd" "SSL_DIR=$ssl_dir" > "$out_dir/$label-build.log" 2>&1 + "$make_cmd" -C "$repo_root" clean > "$tmp_dir/$label-clean.log" 2>&1 + "$make_cmd" -C "$repo_root" build "JERBUILD=$jerbuild_cmd" "SSL_DIR=$ssl_dir" > "$tmp_dir/$label-build.log" 2>&1 mkdir -p "$dest" cp -R "$repo_root/lib" "$dest/lib" - if [ -f "$repo_root/src/.jerbuild-hashes" ]; then - cp "$repo_root/src/.jerbuild-hashes" "$out_dir/$label-jerbuild-hashes" - fi } hash_snapshot() { local root="$1" local output="$2" + if [ ! -d "$root" ]; then + : > "$output" + return 0 + fi (cd "$root" && find . -type f -print | LC_ALL=C sort | while IFS= read -r file; do shasum -a 256 "$file" done) > "$output" } +compare_manifest() { + local name="$1" + local first="$2" + local second="$3" + local raw_diff="$tmp_dir/$name-diff.raw" + if diff -u "$first" "$second" > "$raw_diff"; then + : > "$out_dir/$name-diff.txt" + printf '%s_status=match\n' "$name" >> "$out_dir/result.txt" + return 0 + fi + awk -v name="$name" 'NR == 1 { print "--- first-" name; next } NR == 2 { print "+++ second-" name; next } { print }' "$raw_diff" > "$out_dir/$name-diff.txt" + printf '%s_status=mismatch\n' "$name" >> "$out_dir/result.txt" + return 1 +} + rm -rf "$out_dir" -mkdir -p "$out_dir/first" "$out_dir/second" +mkdir -p "$out_dir" -run_build first "$out_dir/first" -run_build second "$out_dir/second" +run_build first "$tmp_dir/first" +run_build second "$tmp_dir/second" -hash_snapshot "$out_dir/first" "$out_dir/first.sha256" -hash_snapshot "$out_dir/second" "$out_dir/second.sha256" +hash_snapshot "$tmp_dir/first/lib" "$out_dir/first-generated-lib.sha256" +hash_snapshot "$tmp_dir/second/lib" "$out_dir/second-generated-lib.sha256" { printf 'repo=jerboa-https\n' - printf 'jerbuild_command=%s\n' "$jerbuild_cmd" - printf 'ssl_dir=%s\n' "$ssl_dir" + printf 'jerbuild_version=\n' + "$jerbuild_cmd" --version 2>&1 || true + printf 'ssl_dir_status=%s\n' "$([ -d "$ssl_dir" ] && printf present || printf missing)" + printf 'scratch_build_logs_status=not-archived\n' + printf 'scratch_artifacts_status=not-archived\n' + printf 'artifact_archive_status=hashes-only\n' } > "$out_dir/build-env.txt" -if diff -u "$out_dir/first.sha256" "$out_dir/second.sha256" > "$out_dir/hash-diff.txt"; then +: > "$out_dir/result.txt" +overall=0 +compare_manifest generated_lib "$out_dir/first-generated-lib.sha256" "$out_dir/second-generated-lib.sha256" || overall=1 + +if [ "$overall" -eq 0 ]; then { + printf 'artifact_archive_status=hashes-only\n' + printf 'scratch_build_logs_status=not-archived\n' printf 'status=match\n' printf 'detail=two clean generated-library builds matched by SHA-256 manifest\n' - } > "$out_dir/result.txt" + } >> "$out_dir/result.txt" exit 0 fi { + printf 'artifact_archive_status=hashes-only\n' + printf 'scratch_build_logs_status=not-archived\n' printf 'status=mismatch\n' printf 'detail=two clean generated-library builds did not match by SHA-256 manifest\n' printf 'production_blocker=reproducible generated-library evidence remains open until this report shows status=match or an accepted release exception is recorded\n' -} > "$out_dir/result.txt" - -diff -qr "$out_dir/first" "$out_dir/second" > "$out_dir/tree-diff.txt" || true +} >> "$out_dir/result.txt" if [ "$require_match" = "1" ]; then cat "$out_dir/result.txt" >&2 new file mode 100755 --- /dev/null +++ b/scripts/sanitize-evidence.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +if [ "$#" -eq 0 ]; then + set -- dist/release-evidence +fi + +for evidence_dir in "$@"; do + if [ ! -d "$evidence_dir" ]; then + continue + fi + + find "$evidence_dir" -type f -print | while IFS= read -r file; do + perl -0pi -e ' + s#/Users/[^[:space:]\x22\x27()]+#<local-path>#g; + if (defined $ENV{HOME} && length $ENV{HOME}) { + my $home = quotemeta($ENV{HOME}); + s#$home#<home>#g; + } + s#~/mine#<local-path>#g; + s#\$\((HOME)\)/mine#<local-path>#g; + s#users-MacBook-Pro#<host>#g; + s#git\@#git-at-#g; + ' "$file" + done +done --- a/scripts/sbom.sh +++ b/scripts/sbom.sh @@ -14,9 +14,16 @@ ssl_dir="${SSL_DIR:-$repo_root/../jerboa-ssl}" jerboa_source="${JERBOA_SOURCE:-$repo_root/../jerboa}" run_jerbuild() { - ${jerbuild_cmd} "$@" + "$jerbuild_cmd" "$@" } +sanitize_output() { + if [ -x "$repo_root/scripts/sanitize-evidence.sh" ]; then + "$repo_root/scripts/sanitize-evidence.sh" "$out_dir" + fi +} +trap sanitize_output EXIT + hash_tree() { local root="$1" shift @@ -33,7 +40,11 @@ git_report() { local path="$2" { printf 'label=%s\n' "$label" - printf 'path=%s\n' "$path" + if [ -d "$path" ]; then + printf 'path_status=present\n' + else + printf 'path_status=missing\n' + fi if git -C "$path" rev-parse --is-inside-work-tree >/dev/null 2>&1; then printf 'commit=%s\n' "$(git -C "$path" rev-parse HEAD)" printf 'branch=%s\n' "$(git -C "$path" rev-parse --abbrev-ref HEAD 2>/dev/null || true)" @@ -53,12 +64,25 @@ jerboa_home="$(run_jerbuild --jerboa-home 2>/dev/null || true)" { printf 'generated_at_utc=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" printf 'repo=jerboa-https\n' - printf 'repo_root=%s\n' "$repo_root" - printf 'jerbuild_command=%s\n' "$jerbuild_cmd" - printf 'jerboa_home=%s\n' "$jerboa_home" - printf 'ssl_dir=%s\n' "$ssl_dir" - printf 'jerboa_source=%s\n' "$jerboa_source" - printf 'uname=%s\n' "$(uname -a)" + printf 'repo_root_status=present\n' + if [ -n "$jerboa_home" ]; then + printf 'jerboa_home_status=present\n' + else + printf 'jerboa_home_status=missing\n' + fi + if [ -d "$ssl_dir" ]; then + printf 'ssl_dir_status=present\n' + else + printf 'ssl_dir_status=missing\n' + fi + if [ -d "$jerboa_source" ]; then + printf 'jerboa_source_status=present\n' + else + printf 'jerboa_source_status=missing\n' + fi + printf 'os=%s\n' "$(uname -s)" + printf 'kernel_release=%s\n' "$(uname -r)" + printf 'machine=%s\n' "$(uname -m)" printf 'jerbuild_version=\n' run_jerbuild --version 2>&1 || true } > "$out_dir/toolchain.txt" --- a/scripts/soak-evidence.sh +++ b/scripts/soak-evidence.sh @@ -19,9 +19,23 @@ require_target_load_proof="${JHTTPS_REQUIRE_TARGET_LOAD_PROOF:-0}" rm -rf "$out_dir" mkdir -p "$out_dir" +sanitize_output() { + if [ -x "$repo_root/scripts/sanitize-evidence.sh" ]; then + "$repo_root/scripts/sanitize-evidence.sh" "$out_dir" + fi +} +trap sanitize_output EXIT + proof_status_file="$out_dir/proof-status.txt" : > "$proof_status_file" proof_blocked=0 +proof_max_bytes="${JHTTPS_TARGET_PROOF_MAX_BYTES:-65536}" + +proof_contains() { + local pattern=$1 + local file=$2 + LC_ALL=C grep -E -q "$pattern" "$file" +} copy_proof() { local label=$1 @@ -30,7 +44,7 @@ copy_proof() { local status_key=$4 shift 4 local dest="$out_dir/${label}-proof.txt" - local proof_path marker + local proof_path marker bytes secret_pattern private_pattern if [ -z "$source_path" ]; then if [ "$required" = "1" ]; then @@ -53,6 +67,35 @@ copy_proof() { return 0 fi + bytes="$(wc -c < "$proof_path" | tr -d '[:space:]')" + case "$bytes" in + ''|*[!0-9]*) + printf '%s=blocked-invalid-size\n' "$status_key" >> "$proof_status_file" + proof_blocked=1 + return 0 + ;; + *) + if [ "$bytes" -gt "$proof_max_bytes" ]; then + printf '%s=blocked-overlarge\n' "$status_key" >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + ;; + esac + + secret_pattern='(BEGIN [A-Z ]*PRIVATE KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16}|tls[_-]?private[_-]?key[[:space:]]*[:=]|auth[_-]?secret[[:space:]]*[:=]|password[[:space:]]*[:=]|token[[:space:]]*[:=])' + private_pattern='(/Users/|~/mine|\$\(HOME\)/mine|git@|users-MacBook-Pro|uname[[:space:]]+-a)' + if proof_contains "$secret_pattern" "$proof_path"; then + printf '%s=blocked-sensitive-material\n' "$status_key" >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + if proof_contains "$private_pattern" "$proof_path"; then + printf '%s=blocked-private-material\n' "$status_key" >> "$proof_status_file" + proof_blocked=1 + return 0 + fi + for marker in "$@"; do if ! grep -q "$marker" "$proof_path"; then printf '%s=blocked-marker-missing\n' "$status_key" >> "$proof_status_file" @@ -89,10 +132,24 @@ fi printf 'generated_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" printf 'requested_run=%s\n' "$run_requested" printf 'production_min_requests=%s\n' "$production_min_requests" - printf 'target_load_proof_file=%s\n' "$target_load_proof_file" - printf 'jerbuild_command=%s\n' "$jerbuild_cmd" - printf 'ssl_dir=%s\n' "$ssl_dir" - printf 'uname=%s\n' "$(uname -a)" + if [ -n "$target_load_proof_file" ]; then + printf 'target_load_proof_file_status=provided\n' + else + printf 'target_load_proof_file_status=not-provided\n' + fi + if [ -n "$jerbuild_cmd" ]; then + printf 'jerbuild_status=selected\n' + else + printf 'jerbuild_status=missing\n' + fi + if [ -d "$ssl_dir" ]; then + printf 'ssl_dir_status=present\n' + else + printf 'ssl_dir_status=missing\n' + fi + printf 'os=%s\n' "$(uname -s)" + printf 'kernel_release=%s\n' "$(uname -r)" + printf 'machine=%s\n' "$(uname -m)" } > "$out_dir/environment.txt" if [ "$proof_blocked" -ne 0 ]; then old mode 100644 new mode 100755 --- a/scripts/tls-proxy-evidence.sh +++ b/scripts/tls-proxy-evidence.sh @@ -14,11 +14,25 @@ require_target_tls_proxy_proof="${JHTTPS_REQUIRE_TARGET_TLS_PROXY_PROOF:-0}" rm -rf "$out_dir" mkdir -p "$out_dir" +sanitize_output() { + if [ -x "$repo_root/scripts/sanitize-evidence.sh" ]; then + "$repo_root/scripts/sanitize-evidence.sh" "$out_dir" + fi +} +trap sanitize_output EXIT + proof_status=not-run proof_blocked=0 +proof_max_bytes="${JHTTPS_TARGET_PROOF_MAX_BYTES:-65536}" + +proof_contains() { + local pattern=$1 + local file=$2 + LC_ALL=C grep -E -q "$pattern" "$file" +} copy_proof() { - local proof_path marker + local proof_path marker bytes secret_pattern private_pattern if [ -z "$target_tls_proxy_proof_file" ]; then if [ "$require_target_tls_proxy_proof" = "1" ]; then @@ -39,6 +53,35 @@ copy_proof() { return 0 fi + bytes="$(wc -c < "$proof_path" | tr -d '[:space:]')" + case "$bytes" in + ''|*[!0-9]*) + proof_status=blocked-invalid-size + proof_blocked=1 + return 0 + ;; + *) + if [ "$bytes" -gt "$proof_max_bytes" ]; then + proof_status=blocked-overlarge + proof_blocked=1 + return 0 + fi + ;; + esac + + secret_pattern='(BEGIN [A-Z ]*PRIVATE KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16}|tls[_-]?private[_-]?key[[:space:]]*[:=]|auth[_-]?secret[[:space:]]*[:=]|password[[:space:]]*[:=]|token[[:space:]]*[:=])' + private_pattern='(/Users/|~/mine|\$\(HOME\)/mine|git@|users-MacBook-Pro|uname[[:space:]]+-a)' + if proof_contains "$secret_pattern" "$proof_path"; then + proof_status=blocked-sensitive-material + proof_blocked=1 + return 0 + fi + if proof_contains "$private_pattern" "$proof_path"; then + proof_status=blocked-private-material + proof_blocked=1 + return 0 + fi + for marker in \ '^tls_proxy_smoke_status=target-evidence-recorded$' \ '^forwarded_header_policy_status=deny-by-default$' \ @@ -60,9 +103,15 @@ copy_proof { printf 'generated_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" - printf 'target_tls_proxy_proof_file=%s\n' "$target_tls_proxy_proof_file" + if [ -n "$target_tls_proxy_proof_file" ]; then + printf 'target_tls_proxy_proof_file_status=provided\n' + else + printf 'target_tls_proxy_proof_file_status=not-provided\n' + fi printf 'require_target_tls_proxy_proof=%s\n' "$require_target_tls_proxy_proof" - printf 'uname=%s\n' "$(uname -a)" + printf 'os=%s\n' "$(uname -s)" + printf 'kernel_release=%s\n' "$(uname -r)" + printf 'machine=%s\n' "$(uname -m)" } > "$out_dir/environment.txt" target_tls_proxy_status=blocked-not-run --- a/tests/httpd-test.ss +++ b/tests/httpd-test.ss @@ -178,7 +178,7 @@ (http-respond w 200 (list (cons "Content-Type" "text/plain") (cons "X-Method" (http-req-method req))) - (if body body "no body"))))) + (if body (utf8->string body) "no body"))))) (httpd-route "/json" (lambda (req w) (http-respond-json w 200 "{\"status\":\"ok\"}")))