Harden edge runtime and release verification
ober
bd0401346b67f1b1c61a0668021f42611edabc37
--- a/.github/workflows/security-baseline.yml +++ b/.github/workflows/security-baseline.yml @@ -13,27 +13,26 @@ jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Install build tools - run: sudo apt-get update && sudo apt-get install -y build-essential curl ca-certificates git ripgrep pkg-config + run: sudo apt-get update && sudo apt-get install -y build-essential curl ca-certificates git openssh-client ripgrep pkg-config - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable snapshot + with: + toolchain: 1.89.0 - name: Install cargo-audit - run: cargo install cargo-audit --locked + run: cargo install cargo-audit --version 0.22.2 --locked - name: Install jerbuild - run: | - set -eux - curl -fsSL "https://github.com/jerboa-lang/jerboa/releases/download/${JERBOA_VERSION}/jerbuild-linux-x86_64" -o /usr/local/bin/jerbuild - chmod +x /usr/local/bin/jerbuild - env: - JERBOA_VERSION: v0.2.3 + run: support/install-verified-jerbuild.sh /tmp/jerbuild - name: Verify - run: JERBUILD=/usr/local/bin/jerbuild make verify + run: JERBUILD=/tmp/jerbuild make verify - name: Release evidence - run: JERBUILD=/usr/local/bin/jerbuild make release-evidence + run: JERBUILD=/tmp/jerbuild make release-evidence --- a/.jerboa/security.json +++ b/.jerboa/security.json @@ -3,7 +3,7 @@ "repo": "jerboa-edge", "extends": ["jerboa:daemon", "jerboa:ffi", "jerboa:network-service"], "paths": { - "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.{ss,sls,c,h,rs}", "native/**/*.{c,h,rs}", "support/**/*.{c,h,ss,sh}", "Makefile", ".jerbuild"], + "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.{ss,sls,c,h,rs}", "native/**/*.{c,h,rs}", "support/**/*.{c,h,ss,sh,lock}", "Makefile", ".jerbuild", ".github/workflows/*.yml"], "tests": ["test/**", "tests/**", "**/*-test.ss"], "generated": ["build/**", "dist/**", "target/**", "*.so", "*.dylib", "*.wpo"], "vendor": ["vendor/**", "third_party/**"], @@ -12,10 +12,10 @@ "policy": { "failOn": ["critical", "high", "medium"], "imports": { "directChezscheme": "deny-in-project-source" }, - "ffi": { "allowed": true, "nativeFeatures": ["crypto", "wasm"], "denyFeatures": ["tls"], "requireDynamicWindCleanup": true }, + "ffi": { "allowed": true, "nativeFeatures": ["crypto", "wasm", "http"], "denyFeatures": ["tls"], "requireDynamicWindCleanup": true }, "process": { "shellInterpolation": "deny" }, "filesystem": { "allowedWritePaths": ["EDGE_DB_PATH", "temporary embedded-program file"], "requireDocumentedPaths": true }, - "network": { "ingress": "plain-http-behind-external-mtls", "requireTimeouts": true, "requireFailClosedSandbox": true, "requireMtlsForAdmin": true }, + "network": { "bindDefault": "127.0.0.1", "ingress": "plain-http-behind-external-mtls", "applicationAuth": ["webhook-envelope", "read-capability", "operator-capability", "code-capability"], "requireTimeouts": true, "requireFailClosedSandbox": true, "requireMtlsForAdmin": true, "boundedBodyBeforeAllocation": true, "maxConnections": true, "rateLimit": true }, "releaseEvidence": { "linuxRuntime": "current-smoke-recorded-or-target-proof-required", "sustainedLoad": "release-host-sustained-recorded-or-target-proof-required", @@ -26,7 +26,9 @@ "forbiddenEvidenceMaterial": ["host-private-paths", "raw-git-remotes", "private-keys", "credential-shaped-material"] }, "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false }, - "auth": { "rejectUnsignedWebhooks": true, "allowUnsignedOnlyWithExplicitDevFlag": "EDGE_ALLOW_UNSIGNED" } + "auth": { "rejectUnsignedWebhooks": true, "unsignedMode": "none", "binds": ["method", "path", "timestamp", "source", "event-id", "nonce", "body-digest"], "persistentReplayWindow": true, "roleTokensDistinct": true }, + "retention": { "events": "ttl-and-count", "deadLetters": "ttl-and-count", "replayKeys": "ttl-and-count", "watchers": "count", "filters": "count-and-bytes", "handlers": "count-and-bytes", "retries": "global-and-source-type-heap" }, + "supplyChain": { "jsqlite": "commit-and-tree-pinned", "actions": "full-commit-pinned", "jerbuildBootstrap": "signed-manifest-plus-pinned-digest-or-block" } }, "runtimeTargets": ["linux"], "suppressions": [] --- a/.jerbuild +++ b/.jerbuild @@ -3,10 +3,10 @@ ;; The Makefile copies jerbuild's bundled jerboa-native-rs source into ;; .jerboa/jerboa-native-rs before invoking `jerbuild build`; the crate then ;; builds under this repo instead of the shared jerbuild cache target. -;; jerboa-native provides HMAC/constant-time crypto and WASM-backed restricted -;; execution. +;; jerboa-native provides HMAC/constant-time crypto, the HTTP parser shim, and +;; WASM-backed restricted execution. It does not link the TLS feature. -(entry "edge.ss") +(entry "edge-main.ss") (output "jerboa-edge") (libdirs "." "vendor/jsqlite/src") ;; Static binary: a custom main.c sets JERBOA_STATIC=1 so the bundled std @@ -14,8 +14,9 @@ ;; the list below (the linked-in archive members would otherwise be ;; dead-stripped / unreachable via dlsym). (main-c "support/static-main.c") +(extra-sources "support/platform-stubs.c") (ffi-symbols "support/ffi-symbols.gen") (rust-crates (".jerboa/jerboa-native-rs/Cargo.toml" - features: "crypto,wasm" + features: "crypto,wasm,http" no-default-features: #t)) new file mode 100644 --- /dev/null +++ b/.jerbuild-test @@ -0,0 +1,10 @@ +(entry "test/security-regression.ss") +(output "dist/edge-security-test") +(libdirs "." "vendor/jsqlite/src") +(main-c "support/static-main.c") +(extra-sources "support/platform-stubs.c") +(ffi-symbols "support/ffi-symbols.gen") +(rust-crates + (".jerboa/jerboa-native-rs/Cargo.toml" + features: "crypto,wasm,http" + no-default-features: #t)) --- a/Makefile +++ b/Makefile @@ -12,13 +12,15 @@ endif JERBOA_NATIVE_BUNDLE_DIR := $(JH)/jerboa-native-rs NATIVE_DIR ?= $(CURDIR)/.jerboa/jerboa-native-rs NATIVE_A ?= $(NATIVE_DIR)/target/release/libjerboa_native.a -NATIVE_FEATURES ?= crypto,wasm +NATIVE_FEATURES ?= crypto,wasm,http NATIVE_NO_DEFAULT_FEATURES ?= 1 BIN := jerboa-edge BIN_DIR := $(HOME)/.local/bin VENDOR := $(CURDIR)/vendor JSQLITE_REPO ?= $(VENDOR)/jsqlite JSQLITE_URL ?= https://git.sr.ht/~lisp/jsqlite +JSQLITE_COMMIT := 63d737101b3b74bff3b9db75cc35f34bff6df0b7 +JSQLITE_TREE := 79bfcf7a52d233076301c5cba9cd56d1b3e07ebf LIBDIRS := .:$(JH)/lib:$(JSQLITE_REPO)/src EVIDENCE_DIR ?= dist/release-evidence SBOM_DIR ?= dist/sbom @@ -26,15 +28,22 @@ REPRO_DIR ?= dist/reproducibility SOAK_DIR ?= dist/soak SMOKE_PORT ?= $(shell python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' 2>/dev/null || echo 18080) -.PHONY: all build binary run install clean clean-generated ensure-jsqlite ensure-native-src import-check security audit smoke sbom reproducibility-report soak-evidence verify release-evidence +.PHONY: all build binary run install clean clean-generated ensure-jsqlite ensure-native-src import-check unit-test security audit smoke sbom reproducibility-report soak-evidence verify release-evidence all: binary ensure-jsqlite: @if [ ! -f "$(JSQLITE_REPO)/src/jsqlite/api.ss" ]; then \ - mkdir -p "$(VENDOR)"; \ - git clone --depth 1 "$(JSQLITE_URL)" "$(JSQLITE_REPO)"; \ + rm -rf "$(JSQLITE_REPO)"; \ + mkdir -p "$(JSQLITE_REPO)"; \ + git -C "$(JSQLITE_REPO)" init -q; \ + git -C "$(JSQLITE_REPO)" fetch -q --depth 1 "$(JSQLITE_URL)" "$(JSQLITE_COMMIT)"; \ + git -C "$(JSQLITE_REPO)" checkout -q --detach FETCH_HEAD; \ fi + @test "$$(git -C "$(JSQLITE_REPO)" rev-parse HEAD)" = "$(JSQLITE_COMMIT)" || { echo "ERROR: jsqlite commit mismatch" >&2; exit 1; } + @test "$$(git -C "$(JSQLITE_REPO)" rev-parse HEAD^{tree})" = "$(JSQLITE_TREE)" || { echo "ERROR: jsqlite tree mismatch" >&2; exit 1; } + @git -C "$(JSQLITE_REPO)" diff --quiet --exit-code && git -C "$(JSQLITE_REPO)" diff --cached --quiet --exit-code || { echo "ERROR: jsqlite source tree is dirty" >&2; exit 1; } + @test -z "$$(git -C "$(JSQLITE_REPO)" status --porcelain=v1 --untracked-files=all)" || { echo "ERROR: jsqlite source tree contains untracked files" >&2; exit 1; } @test -f "$(JSQLITE_REPO)/src/jsqlite/api.ss" ensure-native-src: @@ -68,7 +77,22 @@ install: binary @echo "Installed $(BIN) to $(BIN_DIR)/$(BIN)" import-check: ensure-jsqlite - $(JERBUILD) exec --libdirs "$(LIBDIRS)" support/import-check.ss + @if [ "$$(uname -s)" != "Linux" ]; then \ + echo "import-check: skipped on $$(uname -s); static unit build covers imports and the runtime target is Linux"; \ + else \ + $(JERBUILD) exec --libdirs "$(LIBDIRS)" support/import-check.ss; \ + fi + +unit-test: binary + $(JERBUILD) build --config .jerbuild-test + EDGE_SECRET='webhook-secret-00000000000000000001' \ + EDGE_READ_TOKEN='read-capability-00000000000000000001' \ + EDGE_OPERATOR_TOKEN='operator-capability-000000000000001' \ + EDGE_CODE_TOKEN='code-capability-00000000000000000001' \ + EDGE_MAX_EVENTS=3 EDGE_MAX_DEAD_LETTERS=2 EDGE_MAX_FILTERS=2 \ + EDGE_MAX_HOT_HANDLERS=2 EDGE_MAX_WATCHERS=2 \ + EDGE_MAX_PENDING_RETRIES=3 EDGE_MAX_RETRIES_PER_SOURCE_TYPE=2 \ + EDGE_MAX_REPLAY_KEYS=8 ./dist/edge-security-test security: clean-generated bash scripts/security-check.sh @@ -99,10 +123,18 @@ smoke: binary exit 0; \ fi; \ log="dist/jerboa-edge-smoke.log"; \ + tmp=$$(mktemp -d "$${TMPDIR:-/tmp}/jedge-smoke.XXXXXXXX"); \ + secret='smoke-webhook-secret-00000000000001'; \ + read_token='smoke-read-capability-00000000000001'; \ + operator_token='smoke-operator-capability-0000000001'; \ + code_token='smoke-code-capability-00000000000001'; \ mkdir -p dist; \ - EDGE_PORT="$(SMOKE_PORT)" EDGE_ALLOW_UNSIGNED=1 ./$(BIN) > "$$log" 2>&1 & \ + EDGE_PORT="$(SMOKE_PORT)" EDGE_DB_PATH="$$tmp/events.db" \ + EDGE_SECRET="$$secret" EDGE_READ_TOKEN="$$read_token" \ + EDGE_OPERATOR_TOKEN="$$operator_token" EDGE_CODE_TOKEN="$$code_token" \ + ./$(BIN) > "$$log" 2>&1 & \ pid=$$!; \ - trap 'kill $$pid >/dev/null 2>&1 || true; wait $$pid >/dev/null 2>&1 || true' EXIT INT TERM; \ + trap 'kill $$pid >/dev/null 2>&1 || true; wait $$pid >/dev/null 2>&1 || true; rm -rf "$$tmp"' EXIT INT TERM; \ ok=0; \ for i in $$(seq 1 50); do \ if curl -fsS "http://127.0.0.1:$(SMOKE_PORT)/health" >/dev/null 2>&1; then ok=1; break; fi; \ @@ -110,8 +142,16 @@ smoke: binary done; \ test "$$ok" = 1; \ curl -fsS "http://127.0.0.1:$(SMOKE_PORT)/health" | grep -q '"status":"ok"'; \ - code=$$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$(SMOKE_PORT)/hooks/smoke" -H 'Content-Type: application/json' -d '{"id":"smoke_1","amount":1}'); \ + code=$$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$(SMOKE_PORT)/api/stats"); \ + test "$$code" = 401; \ + code=$$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:$(SMOKE_PORT)/api/stats" -H "Authorization: Bearer $$read_token"); \ + test "$$code" = 200; \ + body='{"id":"smoke_1","amount":1}'; timestamp=$$(date +%s); source='smoke-source'; nonce='smoke-nonce-00001'; path='/hooks/smoke'; \ + sig=$$(printf '%s' "$$body" | EDGE_SECRET="$$secret" EDGE_SIGN_PATH="$$path" EDGE_SIGN_TIMESTAMP="$$timestamp" EDGE_SIGN_SOURCE="$$source" EDGE_SIGN_EVENT_ID='smoke_1' EDGE_SIGN_NONCE="$$nonce" python3 test/sign-webhook.py); \ + code=$$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$(SMOKE_PORT)$$path" -H 'Content-Type: application/json' -H "x-edge-signature: $$sig" -H "x-edge-timestamp: $$timestamp" -H "x-edge-source: $$source" -H 'x-edge-event-id: smoke_1' -H "x-edge-nonce: $$nonce" -d "$$body"); \ test "$$code" = 202; \ + code=$$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$(SMOKE_PORT)$$path" -H 'Content-Type: application/json' -H "x-edge-signature: $$sig" -H "x-edge-timestamp: $$timestamp" -H "x-edge-source: $$source" -H 'x-edge-event-id: smoke_1' -H "x-edge-nonce: $$nonce" -d "$$body"); \ + test "$$code" = 409; \ echo "smoke: ok" sbom: binary @@ -143,7 +183,7 @@ soak-evidence: SMOKE_PORT="$(SMOKE_PORT)" \ sh scripts/soak-evidence.sh -verify: security audit import-check smoke +verify: security audit import-check unit-test smoke release-evidence: @REPO_ROOT="$(CURDIR)" \ @@ -161,6 +201,7 @@ release-evidence: clean-generated: rm -f $(BIN) $(BIN)-main.c support/ffi-symbols.gen + rm -f dist/edge-security-test dist/edge-security-test-main.c rm -f dist/jerboa-edge-smoke.log find . \( -name '*.so' -o -name '*.wpo' -o -name '*.o' -o -name '*.a' \) -delete 2>/dev/null || true --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # jerboa-edge -`jerboa-edge` is a Jerboa webhook processing daemon. It accepts plain HTTP -from a trusted local or private-network ingress, verifies webhook HMACs, runs -event filters and hot handlers in Jerboa restricted execution, persists optional -state through jsqlite, and exposes health, metrics, event query, dead-letter, +`jerboa-edge` is a Jerboa webhook processing daemon. It binds to loopback by +default, accepts plain HTTP from a trusted ingress, verifies versioned webhook +envelopes with freshness and replay protection, runs +event filters and hot handlers in Jerboa restricted execution, persists bounded +state and replay claims through jsqlite, and exposes health, metrics, event query, dead-letter, and dashboard endpoints. This daemon is not its own internet TLS endpoint. Production deployments must put it behind an external mTLS terminator or service mesh and expose the daemon -only on a private listener. +only on a private listener. External ingress is defense in depth: every route +except `/health` also has an application authentication boundary. ## Build @@ -17,7 +19,8 @@ JERBUILD=/path/to/jerbuild make build ``` The static binary is written to `./jerboa-edge`. The build uses Jerboa native -features `crypto,wasm`; TLS native code is intentionally not linked. +features `crypto,wasm,http`; the `http` feature contains only the bounded +HTTP parser/writev shim, while TLS native code is intentionally not linked. Production runtime target is Linux. The static fiber HTTP server uses Jerboa's Linux epoll-native backend; macOS can build the binary but does not provide the @@ -27,14 +30,38 @@ runtime epoll symbols needed to execute it. ```sh EDGE_SECRET='replace-with-webhook-secret' \ +EDGE_READ_TOKEN='independent-read-capability' \ +EDGE_OPERATOR_TOKEN='independent-operator-capability' \ +EDGE_CODE_TOKEN='independent-code-capability' \ EDGE_PORT=8080 \ EDGE_DB_PATH=/var/lib/jerboa-edge/events.db \ ./jerboa-edge ``` -`EDGE_SECRET` is required for production webhook ingestion. If it is missing, -unsigned webhook requests fail closed. Local development can opt in to unsigned -requests with `EDGE_ALLOW_UNSIGNED=1`. +All four secrets must contain at least 32 UTF-8 bytes and be distinct. +`EDGE_DB_PATH` is mandatory because the bounded replay window must survive a +daemon restart. There is no unsigned mode. A non-loopback `EDGE_BIND` also +requires `EDGE_ALLOW_NON_LOOPBACK_BIND=1` so exposure is never accidental. + +Webhook clients sign this canonical UTF-8 envelope with HMAC-SHA256 and send +the lowercase hex MAC in `x-edge-signature`: + +```text +jerboa-edge-v1\n +POST\n +/hooks/<type>\n +<unix timestamp>\n +<source id>\n +<event id>\n +<nonce>\n +<lowercase SHA-256 of exact body bytes> +``` + +The timestamp, source, event ID, and nonce are supplied in +`x-edge-timestamp`, `x-edge-source`, `x-edge-event-id`, and `x-edge-nonce`. +The JSON body's string `id` must equal the signed ID. API callers use +`Authorization: Bearer <role token>`; read, operator, and code-management +tokens are intentionally not interchangeable. Useful endpoints: @@ -47,6 +74,10 @@ Useful endpoints: - `POST /api/dead-letter/:id/replay` - `GET /dashboard` +The daemon enforces body, connection, request-rate, queue, retry, watcher, +filter, handler, replay, event, dead-letter, count, and TTL bounds internally. +Ingress limits remain an additional outer layer. + ## Security Release gates: @@ -64,6 +95,12 @@ sustained load evidence. A Linux release host can record sustained load with runtime/load proof can be attached with `JEDGE_TARGET_RUNTIME_PROOF_FILE` and `JEDGE_TARGET_LOAD_PROOF_FILE`. +CI executable bootstrap now fails closed unless a signed Jerboa manifest, +pinned digest, and pinned release identity are all present. The checked lock is +currently `blocked-awaiting-authenticated-upstream-release`; publishing and +pinning that upstream artifact is a release blocker, not a reason to restore +the former unverified download. + The security posture is documented in: - `SECURITY.md` --- a/SECURITY.md +++ b/SECURITY.md @@ -14,9 +14,11 @@ Required gates: - `make verify` passes with the intended `JERBUILD`. - Medium-or-higher Jerboa security scanner findings are zero. -- `EDGE_SECRET` is configured in production. +- `EDGE_SECRET`, `EDGE_READ_TOKEN`, `EDGE_OPERATOR_TOKEN`, and + `EDGE_CODE_TOKEN` are distinct and contain at least 32 bytes each. +- `EDGE_DB_PATH` is configured so replay claims survive restart. - The daemon is reachable only behind external mTLS or an equivalent authenticated ingress. -- `EDGE_ALLOW_UNSIGNED` is unset in production. +- Every route except `/health` passes its in-process webhook or capability check. - Persistence paths, credentials, and service account permissions are documented by the deployer. - Dependency and native-code audit output is clean, or accepted risk is recorded in release notes. - Release evidence includes explicit Linux runtime and sustained load status. @@ -29,12 +31,17 @@ Required gates: ## Production Rules - Do not expose `jerboa-edge` directly to the public internet. -- Do not enable the development unsigned-webhook mode in production. +- There is no unsigned webhook mode. Do not add one. +- Do not reuse capability tokens between read, operator, code-management, or webhook roles. +- Keep the loopback default. Non-loopback binding is an explicit deployment decision. - Do not commit secrets, private keys, tokens, production `.env` files, hostnames, customer data, or database files. - Do not add in-process TLS without a threat model update, scanner coverage, and cleanup tests for native handles. - Do not add broad direct `(chezscheme)` imports in project source. - Do not add string-built SQL, shell interpolation, or bare Scheme `read` on untrusted input. - Keep restricted user filters and hot handlers behind Jerboa restricted execution and timeouts. +- Keep filter failures fail closed and distinguish error/timeout from a policy rejection. +- Do not replace fixed-capacity rings/retry heap with unbounded maps, lists, or sleeping threads. +- Do not execute a downloaded Jerboa bootstrap unless its digest and signed manifest verify against the checked-in release identity. `status=blocked-awaiting-authenticated-upstream-release` is a release blocker. ## Reporting --- a/bench-edge.sh +++ b/bench-edge.sh @@ -1,67 +1,51 @@ -#!/bin/bash -# bench-edge.sh — Benchmark Jerboa Edge throughput -# -# Requires: wrk (apt install wrk / brew install wrk) -# Usage: Start the server first (make run), then run this script. - +#!/usr/bin/env bash +# Benchmark the public health path and authenticated webhook ingestion. +# Start the Linux server with the same EDGE_* environment first. set -euo pipefail -PORT=${EDGE_PORT:-8080} -BASE="http://localhost:${PORT}" -DURATION=${BENCH_DURATION:-10s} -THREADS=${BENCH_THREADS:-4} -CONNECTIONS=${BENCH_CONNECTIONS:-100} - -# Verify server is up -if ! curl -sf "$BASE/health" > /dev/null 2>&1; then - echo "Error: Server not responding at $BASE/health" - echo "Start with: make run" - exit 1 +: "${EDGE_SECRET:?set EDGE_SECRET to the daemon webhook secret}" +: "${EDGE_READ_TOKEN:?set EDGE_READ_TOKEN to the daemon read capability}" + +host=${EDGE_HOST:-127.0.0.1} +port=${EDGE_PORT:-8080} +base="http://${host}:${port}" +duration=${BENCH_DURATION_SECONDS:-10} +threads=${BENCH_THREADS:-4} +connections=${BENCH_CONNECTIONS:-100} +concurrency=${BENCH_SIGNED_CONCURRENCY:-8} + +case "$duration" in + ''|*[!0-9]*) echo "BENCH_DURATION_SECONDS must be an integer" >&2; exit 2 ;; +esac +case "$concurrency" in + ''|*[!0-9]*) echo "BENCH_SIGNED_CONCURRENCY must be an integer" >&2; exit 2 ;; +esac +if (( duration < 1 || concurrency < 1 )); then + echo "benchmark duration and concurrency must be positive" >&2 + exit 2 fi -echo "═══════════════════════════════════════════════" -echo " Jerboa Edge Benchmark" -echo "═══════════════════════════════════════════════" -echo " target: $BASE" -echo " duration: $DURATION" -echo " threads: $THREADS" -echo " connections: $CONNECTIONS" -echo "" - -# Create Lua script for POST requests with unique IDs -cat > /tmp/edge-bench-post.lua << 'LUAEOF' -local counter = 0 +for command in curl wrk python3; do + command -v "$command" >/dev/null 2>&1 || { + echo "required command not found: $command" >&2 + exit 1 + } +done -request = function() - counter = counter + 1 - local body = string.format('{"id":"bench_%d_%d","amount":%d}', - counter, math.random(1000000), math.random(10000)) - return wrk.format("POST", "/hooks/payment.completed", - {["Content-Type"] = "application/json"}, body) -end -LUAEOF - -# ── GET /health (baseline) ──────────────────────────────────── -echo "--- GET /health (baseline) ---" -wrk -t"$THREADS" -c"$CONNECTIONS" -d"$DURATION" "$BASE/health" -echo "" - -# ── POST /hooks/:type (webhook ingestion) ───────────────────── -echo "--- POST /hooks/payment.completed ---" -wrk -t"$THREADS" -c"$CONNECTIONS" -d"$DURATION" -s /tmp/edge-bench-post.lua "$BASE/hooks/payment.completed" -echo "" +if ! curl -fsS "$base/health" >/dev/null; then + echo "server is not responding at $base/health" >&2 + exit 1 +fi -# ── GET /api/stats (read path) ──────────────────────────────── -echo "--- GET /api/stats ---" -wrk -t"$THREADS" -c"$CONNECTIONS" -d"$DURATION" "$BASE/api/stats" -echo "" +printf 'Jerboa Edge benchmark\n' +printf 'target: %s\nduration: %ss\n\n' "$base" "$duration" -# Cleanup -rm -f /tmp/edge-bench-post.lua +printf '%s\n' '--- unauthenticated GET /health baseline ---' +wrk -t"$threads" -c"$connections" -d"${duration}s" "$base/health" -# Final stats -echo "═══════════════════════════════════════════════" -echo " Post-benchmark stats:" -curl -s "$BASE/api/stats" | python3 -m json.tool 2>/dev/null || curl -s "$BASE/api/stats" -echo "" -echo "═══════════════════════════════════════════════" +printf '\n%s\n' '--- freshly signed POST /hooks/payment.completed ---' +# The Python driver generates a unique route-bound MAC, event ID, and nonce for +# every request. It keeps secrets in the environment instead of argv or files. +EDGE_BENCH_HOST="$host" EDGE_BENCH_PORT="$port" \ + EDGE_BENCH_DURATION="$duration" EDGE_BENCH_CONCURRENCY="$concurrency" \ + python3 test/benchmark-signed.py --- a/docs/deployment-security.md +++ b/docs/deployment-security.md @@ -4,8 +4,11 @@ Run `jerboa-edge` behind an external mTLS terminator, service mesh sidecar, or reverse proxy that enforces client authentication, request size limits, rate -limits, access logging, and TLS policy. Bind the daemon to a private interface -or loopback listener. +limits, access logging, and TLS policy. These are outer controls: the daemon +also enforces a pre-allocation body cap, connection admission, fixed-window +request rates, bounded queues/state, and application authentication. It binds +to `127.0.0.1` unless `EDGE_BIND` is set; a non-loopback value additionally +requires `EDGE_ALLOW_NON_LOOPBACK_BIND=1`. The daemon intentionally does not link Jerboa native TLS support. This avoids a second in-process HTTP/TLS parser and keeps TLS certificate/key material out of @@ -22,20 +25,27 @@ bundle. ## Authentication -Set `EDGE_SECRET` in production. Webhook requests without a valid `x-signature` -are rejected when a secret is configured. If `EDGE_SECRET` is absent, unsigned -webhooks are rejected unless `EDGE_ALLOW_UNSIGNED=1` is explicitly set for local -development. - -Do not set `EDGE_ALLOW_UNSIGNED` in production. +Set four independent secrets: `EDGE_SECRET`, `EDGE_READ_TOKEN`, +`EDGE_OPERATOR_TOKEN`, and `EDGE_CODE_TOKEN`. The daemon refuses startup unless +all are distinct and at least 32 UTF-8 bytes. Webhooks require a lowercase +HMAC-SHA256 over the versioned method/path/timestamp/source/event-ID/nonce/body +digest envelope. Timestamps have a bounded skew, IDs must match the body, and +claims are retained in SQLite for the replay window. All query/dashboard, +operator/replay/metrics, and code-management routes require their exact Bearer +capability. Only `/health` is intentionally unauthenticated. ## Persistence -`EDGE_DB_PATH` enables jsqlite persistence. The deployer should place this path +`EDGE_DB_PATH` is required for restart-resistant replay protection. Place it on a dedicated directory owned by the service account with no world-readable permissions. Backups and logs containing payload data should follow the same data handling policy as webhook payloads. +Event, dead-letter, and replay tables are pruned by TTL and count. In-memory +event/dead-letter rings, retry heap, source/type retry counts, filters, handlers, +watchers, connections, request bodies, and ingest queue all have hard caps. +Tune only within the range checks documented by startup errors. + ## Process Sandboxing Use a dedicated service account. For systemd, prefer: --- a/docs/ffi-boundary.md +++ b/docs/ffi-boundary.md @@ -9,10 +9,17 @@ The `.jerbuild` configuration enables only: - `crypto`: HMAC-SHA256 and constant-time comparison. - `wasm`: Jerboa restricted execution support. +- `http`: `httparse` request metadata and the checked scatter/gather writer; + this feature is split from TLS so an HTTP-only daemon does not link rustls. The TLS feature is intentionally disabled. Production TLS and client authentication must be supplied by external ingress. +On non-Linux build hosts, `support/platform-stubs.c` supplies fail-closed epoll +symbols solely so the static image can be imported for unit tests. Every poller +operation returns failure. Linux artifacts do not compile those definitions and +link the real Rust epoll/eventfd implementation. + ## Static Launcher `support/static-main.c` embeds the Scheme program boot data in a temporary file @@ -23,6 +30,9 @@ and invokes Chez through `Sscheme_program`. The launcher: - writes boot data with a checked `fwrite_all` loop, - closes with checked `fclose`, - unlinks the temporary file after the Scheme program starts. +- registers the narrow, typed libc socket/signal surface before Scheme library + initialization; variadic `fcntl` is registered from its real declaration, + not emitted through a guessed generic prototype. ## FFI Symbol List @@ -32,6 +42,7 @@ because it is generated and platform-specific. ## SQLite Boundary -`jsqlite` is vendored under `vendor/` during builds. Prepared statements are +`jsqlite` is fetched at the commit and tree pinned in `Makefile`; a moved +default branch is never consumed. Prepared statements are finalized with `dynamic-wind`, and database handles are closed on failed open and graceful shutdown. --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -9,7 +9,7 @@ The directory includes: - security, audit, import-check, and smoke-test logs, - Cargo metadata for the bundled Jerboa native crate when available, - SBOM/dependency manifests for source, support scripts, vendored jsqlite, - the repo-local no-default `crypto,wasm` native crate copy, binary linkage, + the repo-local no-default `crypto,wasm,http` native crate copy, binary linkage, and native artifact hashes, - repeated clean-build reproducibility reports for the standalone binary, generated FFI symbol list, native archive, and source manifest. The @@ -28,6 +28,14 @@ Target proof files are rejected before copy when they are over `JEDGE_TARGET_PROOF_MAX_BYTES` bytes or contain private host paths, raw `git@` remotes, private-key blocks, or credential-shaped material. +The SBOM records the exact jsqlite commit and tree, native Cargo lock/metadata, +workflow/support hashes, selected native features, binary linkage, and the +verified Jerbuild digest when CI bootstrap is available. Release promotion is +blocked while `support/jerbuild-bootstrap.lock` records +`blocked-awaiting-authenticated-upstream-release`; only a producer-signed +manifest verified against `support/jerboa-release-signers` may change it to +`ready`. + Release evidence is generated from the same gates used by CI: ```sh --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -3,7 +3,7 @@ ## Scope `jerboa-edge` receives webhooks from a trusted ingress, queues events, runs -filters and handlers, stores optional SQLite-backed state, and exposes query, +filters and handlers, stores bounded SQLite-backed state, and exposes query, metrics, health, dead-letter, and dashboard endpoints. ## Assets @@ -16,27 +16,40 @@ metrics, health, dead-letter, and dashboard endpoints. ## Trust Boundaries -- External clients terminate at an mTLS ingress before reaching this daemon. +- External clients terminate at an mTLS ingress before reaching this daemon; + in-process authentication remains mandatory if that boundary is bypassed. - `jerboa-edge` listens on plain HTTP only inside the trusted runtime boundary. -- HMAC verification authenticates webhook payloads when `EDGE_SECRET` is set. +- A versioned HMAC envelope binds method, route, freshness, source, stable event + ID, nonce, and exact body digest. SQLite persists bounded replay claims. +- Independent Bearer capabilities separate read, operator, and code-management authority. - User-provided filter and handler code crosses into Jerboa restricted execution. - jsqlite persistence crosses into a local filesystem/database boundary. -- Jerboa native FFI is limited to `crypto,wasm` for this daemon. +- Jerboa native FFI is limited to `crypto,wasm,http` for this daemon. ## Primary Threats And Controls -- Unauthorized webhook submission: production requires `EDGE_SECRET`; missing secrets fail closed. -- Accidental unsigned mode: `EDGE_ALLOW_UNSIGNED=1` is explicit development-only behavior and is checked by the security gate. -- Public daemon exposure: production requires external mTLS or equivalent authenticated ingress. +- Unauthorized webhook submission: startup requires a strong webhook secret and + strict signed-envelope fields; there is no unsigned mode. +- Replay/route substitution: timestamp skew, exact route binding, stable body ID, + nonce, and restart-persistent replay claims fail closed. +- Public daemon exposure: loopback is the default, non-loopback binding is + explicit, and every non-health route has application authentication in + addition to the required external mTLS boundary. - Code execution through filters or hot handlers: code runs through `restricted-eval-string` with Jerboa restricted bindings and timeouts. - SQL injection: SQL statements are literals and event values are bound parameters. - SQLite handle leaks: prepared statements are finalized with `dynamic-wind`; database handles close on failed open and shutdown. - Native handle leaks: in-process TLS is removed; the remaining static launcher uses checked stdio writes and closes. - Worker crash loss: worker actors run under one-for-one supervision and failed events retry before dead-lettering. +- Resource exhaustion: parsing rejects ambiguous/oversized lengths before body + allocation; connections/rates/queues/state/code/watchers and persistence have + fixed caps and TTLs; one bounded timer heap owns all retries. +- Filter bypass: reject, timeout, and error are distinct and timeout/error never + silently become acceptance. ## Residual Risk -- The daemon depends on the deployer to enforce mTLS, network ACLs, request size limits, rate limits, and process sandboxing. +- The daemon still depends on the deployer for mTLS, network ACLs, and process + sandboxing, but no longer delegates its only authentication or resource limits there. - Restricted execution is still hostile-input-facing and must be reviewed whenever new bindings are added. - The smoke test validates basic runtime behavior on Linux, not sustained load, fuzzing, or full ingress policy. Release evidence must record explicit @@ -44,3 +57,6 @@ metrics, health, dead-letter, and dashboard endpoints. can be attached and hashed, but `load_status=blocked-not-run`, `linux_runtime_status=blocked-*`, or `status=blocked-target-proof` remains a production blocker. +- CI bootstrap remains deliberately blocked until the Jerboa producer publishes + a signed release manifest and stable public identity for the new HTTP feature. + The former mutable, unverified executable download is not an allowed fallback. new file mode 100644 --- /dev/null +++ b/edge-main.ss @@ -0,0 +1,3 @@ +#!chezscheme +(import (jerboa prelude) (edge)) +(run-edge) --- a/edge.ss +++ b/edge.ss @@ -11,19 +11,36 @@ ;;; - Prometheus metrics (/metrics endpoint) ;;; - Structured JSON logging (greppable by Loki/Splunk) ;;; - Graceful SIGTERM shutdown (drain, flush jsqlite WAL) -;;; - Optional jsqlite persistence (EDGE_DB_PATH) -;;; - Production deployment behind an external mTLS terminator +;;; - Mandatory jsqlite replay persistence (EDGE_DB_PATH) +;;; - Application-layer capability tokens plus external mTLS ;;; ;;; Zero external dependencies. Everything is Jerboa stdlib. ;;; ;;; Run: make run -;;; Test: curl -X POST localhost:8080/hooks/payment \ -;;; -d '{"id":"evt_1","amount":4999}' -;;; curl localhost:8080/api/stats -;;; curl localhost:8080/metrics - -(import (except (jerboa prelude) +;;; Test: make unit-test && make smoke + +(library (edge) + (export + run-edge + bytes->hex strict-hex->bytes safe-token? strict-decimal + canonical-webhook-envelope webhook-signature verify-webhook-signature + claim-replay-key! request-authorized? protect-route + bounded-request-body parse-json-object + restore-event! restore-dead-letter! get-event event-stats get-dead-letter-list + eval-user-filter apply-user-filters register-hot-handler! + register-watcher! unregister-watcher! + schedule-retry! retry-due-jobs + handle-webhook handle-add-filter handle-register-handler + respond-json response? response-status response-body + make-request make-router route-post router-dispatch) + + (import (except (scheme) make-hash-table hash-table?) + (except (jerboa prelude) cpu-count or-else + atom? path-absolute? with-input-from-string fprintf + last-pair iota path-extension meta printf sort 1+ 1- + format define-values make-time make-date with-output-to-string + partition sort! json-object->string string->json-object write-json read-json tcp-write-string tcp-write tcp-read tcp-close tcp-accept tcp-listen tcp-connect @@ -37,7 +54,7 @@ (std stm) (std transducer) (std text json) - (std crypto native) + (std crypto native-rust) (std pmap) (std security restrict) (std misc timeout) @@ -76,7 +93,7 @@ [(= rc jconst:SQLITE_DONE) (reverse acc)] [else - (reverse acc)]))))) + (error 'sqlite-query "sqlite step failed" rc)]))))) (lambda () (when stmt (guard (e [(condition? e) (void)]) @@ -86,18 +103,88 @@ ;; Configuration ;; ═══════════════════════════════════════════════════════════════ -(def *port* (or (and (getenv "EDGE_PORT") (string->number (getenv "EDGE_PORT"))) 8080)) -(def *workers* (or (and (getenv "EDGE_WORKERS") (string->number (getenv "EDGE_WORKERS"))) 4)) -(def *secret* (or (getenv "EDGE_SECRET") "")) -;; P2.2 fix: empty *secret* must NOT silently accept all requests. -;; To run without HMAC for local development, set EDGE_ALLOW_UNSIGNED=1 -;; explicitly. Any other unset/empty value fails closed. -(def *allow-unsigned?* - (let ([v (getenv "EDGE_ALLOW_UNSIGNED")]) - (and v (or (string=? v "1") (string=? v "true") (string=? v "yes"))))) -(def *db-path* (getenv "EDGE_DB_PATH")) ;; SQLite file path, or #f for in-memory only -(def (secret-configured?) - (> (string-length *secret*) 0)) +(def (env-positive-integer name default minimum maximum) + (let* ([raw (getenv name)] + [n (and raw (string->number raw))]) + (if (not raw) + default + (if (and (integer? n) (exact? n) (<= minimum n) (<= n maximum)) + n + (error 'jerboa-edge "invalid bounded positive integer setting" name))))) + +(def (env-true? name) + (let ([v (getenv name)]) + (and (string? v) + (member (string-downcase v) '("1" "true" "yes" "on")) + #t))) + +(def *port* (env-positive-integer "EDGE_PORT" 8080 1 65535)) +(def *workers* (env-positive-integer "EDGE_WORKERS" 4 1 64)) +(def *bind-host* (or (getenv "EDGE_BIND") "127.0.0.1")) +(def *max-connections* (env-positive-integer "EDGE_MAX_CONNECTIONS" 128 1 4096)) +(def *secret* (or (getenv "EDGE_SECRET") "")) +(def *read-token* (or (getenv "EDGE_READ_TOKEN") "")) +(def *operator-token* (or (getenv "EDGE_OPERATOR_TOKEN") "")) +(def *code-token* (or (getenv "EDGE_CODE_TOKEN") "")) +(def *db-path* (getenv "EDGE_DB_PATH")) + +;; Resource policy. Bounds are deliberately finite even when persistence is +;; disabled; environment overrides are themselves range checked. +(def *max-body-bytes* (env-positive-integer "EDGE_MAX_BODY_BYTES" 262144 1024 1048576)) +(def *max-admin-body-bytes* (env-positive-integer "EDGE_MAX_ADMIN_BODY_BYTES" 65536 1024 262144)) +(def *max-events* (env-positive-integer "EDGE_MAX_EVENTS" 10000 1 1000000)) +(def *max-dead-letters* (env-positive-integer "EDGE_MAX_DEAD_LETTERS" 1000 1 100000)) +(def *max-filters* (env-positive-integer "EDGE_MAX_FILTERS" 64 1 1024)) +(def *max-hot-handlers* (env-positive-integer "EDGE_MAX_HOT_HANDLERS" 64 1 1024)) +(def *max-watchers* (env-positive-integer "EDGE_MAX_WATCHERS" 128 1 4096)) +(def *max-pending-retries* (env-positive-integer "EDGE_MAX_PENDING_RETRIES" 1024 1 65536)) +(def *max-retries-per-key* (env-positive-integer "EDGE_MAX_RETRIES_PER_SOURCE_TYPE" 64 1 4096)) +(def *max-replay-keys* (env-positive-integer "EDGE_MAX_REPLAY_KEYS" 20000 1 1000000)) +(def *replay-window-seconds* (env-positive-integer "EDGE_REPLAY_WINDOW_SECONDS" 300 30 3600)) +(def *event-ttl-seconds* (env-positive-integer "EDGE_EVENT_TTL_SECONDS" 86400 60 31536000)) +(def *dead-ttl-seconds* (env-positive-integer "EDGE_DEAD_LETTER_TTL_SECONDS" 604800 60 31536000)) +(def *max-requests-per-second* (env-positive-integer "EDGE_MAX_REQUESTS_PER_SECOND" 200 1 100000)) + +(def (loopback-host? host) + (or (string=? host "127.0.0.1") + (string=? host "::1") + (string=? host "localhost"))) + +(def (valid-secret? secret) + (and (string? secret) + (>= (bytevector-length (string->utf8 secret)) 32) + (<= (bytevector-length (string->utf8 secret)) 4096) + (let ([bytes (string->utf8 secret)]) + (let loop ([i 0]) + (or (= i (bytevector-length bytes)) + (let ([b (bytevector-u8-ref bytes i)]) + (and (> b 32) (not (= b 127)) (loop (+ i 1))))))))) + +(def (secret-configured?) (valid-secret? *secret*)) + +(def (validate-runtime-config!) + (unless (valid-secret? *secret*) + (error 'jerboa-edge "EDGE_SECRET must contain 32 to 4096 UTF-8 bytes")) + (for-each + (lambda (entry) + (unless (valid-secret? (cdr entry)) + (error 'jerboa-edge "capability token must contain 32 to 4096 UTF-8 bytes" (car entry)))) + `((EDGE_READ_TOKEN . ,*read-token*) + (EDGE_OPERATOR_TOKEN . ,*operator-token*) + (EDGE_CODE_TOKEN . ,*code-token*))) + (when (or (string=? *secret* *read-token*) + (string=? *secret* *operator-token*) + (string=? *secret* *code-token*) + (string=? *read-token* *operator-token*) + (string=? *read-token* *code-token*) + (string=? *operator-token* *code-token*)) + (error 'jerboa-edge "webhook and capability secrets must be distinct")) + (unless (and (string? *db-path*) (> (string-length *db-path*) 0)) + (error 'jerboa-edge + "EDGE_DB_PATH is required so replay identity survives daemon restart")) + (unless (or (loopback-host? *bind-host*) (env-true? "EDGE_ALLOW_NON_LOOPBACK_BIND")) + (error 'jerboa-edge + "non-loopback EDGE_BIND requires explicit EDGE_ALLOW_NON_LOOPBACK_BIND=1" *bind-host*))) ;; ═══════════════════════════════════════════════════════════════ ;; Prometheus Metrics (Phase 3.4) @@ -140,16 +227,18 @@ ;; ═══════════════════════════════════════════════════════════════ ;; jsqlite Persistence (Phase 3.3) ;; -;; Optional: set EDGE_DB_PATH to a file path to enable persistence. +;; EDGE_DB_PATH is mandatory so replay identities survive daemon restarts. ;; On startup, existing events and dead-letters are loaded into STM. ;; Every store-result! / store-dead-letter! also writes to jsqlite. ;; WAL mode is used for concurrent reads with no reader blocking. ;; -;; All DB writes are fire-and-forget: guarded with (guard (e [#t (void)])) -;; so a SQLite error never crashes a worker. +;; Event/dead-letter history writes remain best effort; replay-identity writes +;; are fail closed because accepting a duplicate is a security violation. ;; ═══════════════════════════════════════════════════════════════ (def *db* #f) ;; jsqlite handle — #f when EDGE_DB_PATH is not set +(def *db-mutex* (make-mutex)) +(def *db-write-count* 0) (def (db-init!) (when *db-path* @@ -159,42 +248,113 @@ (lambda () (set! db (sqlite-open *db-path*)) (sqlite-exec db "PRAGMA journal_mode=WAL") + (sqlite-exec db "PRAGMA synchronous=FULL") (sqlite-exec db "CREATE TABLE IF NOT EXISTS events (id TEXT PRIMARY KEY, type TEXT, status TEXT, received TEXT, processed TEXT)") (sqlite-exec db "CREATE TABLE IF NOT EXISTS dead_letters (id TEXT PRIMARY KEY, type TEXT, reason TEXT, retries INTEGER, ts INTEGER)") + (sqlite-exec db "CREATE TABLE IF NOT EXISTS replay_keys (replay_key TEXT PRIMARY KEY, expires INTEGER NOT NULL)") + (sqlite-exec db "CREATE INDEX IF NOT EXISTS events_processed_idx ON events(processed)") + (sqlite-exec db "CREATE INDEX IF NOT EXISTS dead_letters_ts_idx ON dead_letters(ts)") + (sqlite-exec db "CREATE INDEX IF NOT EXISTS replay_keys_expires_idx ON replay_keys(expires)") (set! *db* db) (set! db #f) (jlog "sqlite opened" "path" *db-path*)) (lambda () (when db - (guard (e [#t (void)]) - (db-close-handle! db)))))))) + (db-close-handle! db))))))) + +(def (%db-prune! now) + (sqlite-execute *db* "DELETE FROM events WHERE CAST(processed AS INTEGER) < ?" + (- now *event-ttl-seconds*)) + (sqlite-execute *db* + "DELETE FROM events WHERE id NOT IN (SELECT id FROM events ORDER BY CAST(processed AS INTEGER) DESC LIMIT ?)" + *max-events*) + (sqlite-execute *db* "DELETE FROM dead_letters WHERE ts < ?" (- now *dead-ttl-seconds*)) + (sqlite-execute *db* + "DELETE FROM dead_letters WHERE id NOT IN (SELECT id FROM dead_letters ORDER BY ts DESC LIMIT ?)" + *max-dead-letters*) + (sqlite-execute *db* "DELETE FROM replay_keys WHERE expires < ?" now) + (sqlite-execute *db* + "DELETE FROM replay_keys WHERE replay_key NOT IN (SELECT replay_key FROM replay_keys ORDER BY expires DESC LIMIT ?)" + *max-replay-keys*)) + +(def (db-note-write-and-prune!) + (set! *db-write-count* (+ *db-write-count* 1)) + (when (>= *db-write-count* 100) + (set! *db-write-count* 0) + (%db-prune! (time-second (current-time))))) (def (db-store-event! event status) (when *db* - (guard (e [#t (void)]) - (sqlite-execute *db* - "INSERT OR REPLACE INTO events VALUES (?,?,?,?,?)" - (hash-ref event "id" "") - (hash-ref event "type" "unknown") - status - (hash-ref event "received" "") - (format "~a" (time-second (current-time))))))) + (with-mutex *db-mutex* + (guard (e [(condition? e) (void)]) + (sqlite-execute *db* + "INSERT OR REPLACE INTO events VALUES (?,?,?,?,?)" + (hash-ref event "id" "") + (hash-ref event "type" "unknown") + status + (hash-ref event "received" "") + (format "~a" (time-second (current-time)))) + (db-note-write-and-prune!))))) (def (db-store-dead-letter! event reason) (when *db* - (guard (e [#t (void)]) - (sqlite-execute *db* - "INSERT OR REPLACE INTO dead_letters VALUES (?,?,?,?,?)" - (hash-ref event "id" "") - (hash-ref event "type" "unknown") - reason - (hash-ref event "retry-count" 0) - (time-second (current-time)))))) + (with-mutex *db-mutex* + (guard (e [(condition? e) (void)]) + (sqlite-execute *db* + "INSERT OR REPLACE INTO dead_letters VALUES (?,?,?,?,?)" + (hash-ref event "id" "") + (hash-ref event "type" "unknown") + reason + (hash-ref event "retry-count" 0) + (time-second (current-time))) + (db-note-write-and-prune!))))) + +(def (db-delete-dead-letter! id) + (when *db* + (with-mutex *db-mutex* + (guard (e [(condition? e) (void)]) + (sqlite-execute *db* "DELETE FROM dead_letters WHERE id = ?" id))))) + +;; Must be called while the in-process replay mutex is held. The database is +;; the restart-resistant half of replay protection; a duplicate key is never +;; accepted merely because the daemon restarted. +(def (db-claim-replay! replay-key expires now) + (if (not *db*) + ;; Direct unit tests may use the bounded in-memory cache without selecting + ;; a DB. A configured-but-unavailable DB always fails closed. + (not *db-path*) + (with-mutex *db-mutex* + (guard (e [(condition? e) #f]) + ;; INSERT OR IGNORE is the atomic claim. Unlike SELECT followed by + ;; REPLACE, it also rejects a concurrent claimant in another process. + (sqlite-execute *db* + "DELETE FROM replay_keys WHERE replay_key = ? AND expires < ?" + replay-key now) + (sqlite-execute *db* + "INSERT OR IGNORE INTO replay_keys(replay_key, expires) VALUES (?,?)" + replay-key expires) + (let* ([rows (sqlite-query *db* "SELECT changes() AS inserted")] + [claimed? (and (pair? rows) + (= (cdr (assoc "inserted" (car rows))) 1))]) + (when claimed? (db-note-write-and-prune!)) + claimed?))))) (def (db-load-events!) - ;; Replay persisted events into STM state on startup. + ;; Restore only the bounded, unexpired retention window into memory. (when *db* - (let ([rows (sqlite-query *db* "SELECT id, type, status, received, processed FROM events")]) + (let* ([now (time-second (current-time))] + [rows #f] + [dead #f]) + (with-mutex *db-mutex* + (%db-prune! now) + (set! rows + (sqlite-query *db* + "SELECT id, type, status, received, processed FROM events WHERE CAST(processed AS INTEGER) >= ? ORDER BY CAST(processed AS INTEGER) DESC LIMIT ?" + (- now *event-ttl-seconds*) *max-events*)) + (set! dead + (sqlite-query *db* + "SELECT id, type, reason, retries, ts FROM dead_letters WHERE ts >= ? ORDER BY ts DESC LIMIT ?" + (- now *dead-ttl-seconds*) *max-dead-letters*))) (for-each (lambda (row) (let ([id (cdr (assoc "id" row))] @@ -203,9 +363,8 @@ ("status" . ,(cdr (assoc "status" row))) ("received" . ,(cdr (assoc "received" row))) ("processed" . ,(cdr (assoc "processed" row)))))]) - (dosync (alter events-ref (lambda (st) (persistent-map-set st id ht)))))) - rows)) - (let ([dead (sqlite-query *db* "SELECT id, type, reason, retries FROM dead_letters")]) + (restore-event! id ht))) + (reverse rows)) (for-each (lambda (row) (let ([id (cdr (assoc "id" row))] @@ -213,22 +372,25 @@ `(("id" . ,(cdr (assoc "id" row)))