Security hardening and release readiness
ober
a58fdbe6ed9f5fe185c40014144188de0529f4e0
new file mode 100644 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +env: + JERBOA_VERSION: v0.2.3 + JERBUILD: ${{ github.workspace }}/.jerboa/bin/jerbuild + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system tools + run: | + set -eu + sudo apt-get update + sudo apt-get install -y --no-install-recommends build-essential curl ca-certificates + + - name: Install cargo-audit + run: cargo install cargo-audit --locked + + - name: Install Jerboa toolchain + run: sh support/ensure-jerboa.sh "$JERBOA_VERSION" .jerboa/bin + + - name: Verify + run: make verify + + - name: Release evidence + run: make release-evidence new file mode 100644 --- /dev/null +++ b/.github/workflows/security-baseline.yml @@ -0,0 +1,35 @@ +name: Security Baseline + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + baseline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Required release files + run: | + set -eu + test -f LICENSE + test -f SECURITY.md + test -f .gitignore + find . -maxdepth 1 -iname "README*" -type f | grep -q . + + - name: High-confidence secret scan + run: | + set -eu + pattern="(BEGIN (RSA|OPENSSH|EC|DSA|PRIVATE) KEY|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-(ant-api03|proj|svcacct)-[A-Za-z0-9_-]{30,}|AKIA[0-9A-Z]{16})" + matches="$(git grep -n -I -E "$pattern" -- . ":!*.png" ":!*.jpg" ":!*.jpeg" ":!*.gif" ":!*.so" ":!*.dylib" ":!*.o" ":!*.a" ":!*.boot" ":!*.tar.gz" || true)" + if [ -n "$matches" ]; then + echo "$matches" + echo "High-confidence secret pattern found." + exit 1 + fi --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ *.so *.wpo *.fasl +dist/ +.jerboa/bin/ +.jerboa/native-target/ # Host-specific symlink created by `make setup` (target path differs per machine) lib/libjerboa_native.so --- a/.jerboa/security.json +++ b/.jerboa/security.json @@ -1,9 +1,9 @@ { "version": 1, "repo": "jerboa-db", - "extends": ["jerboa:library", "jerboa:parser"], + "extends": ["jerboa:library", "jerboa:parser", "jerboa:storage"], "paths": { - "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.ss", "src/**/*.sls", "Makefile"], + "production": ["*.ss", "*.sls", "bin/**/*.ss", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.ss", "src/**/*.sls", "support/import-check.ss", "Makefile"], "tests": ["test/**", "tests/**", "**/*-test.ss", "bench/**", "benchmarks/**"], "generated": ["build/**", "dist/**", "target/**"], "vendor": ["vendor/**", "third_party/**"], @@ -16,7 +16,47 @@ "process": { "shellInterpolation": "deny" }, "network": { "requireTimeouts": false }, "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false }, - "parser": { "requireLimits": true } + "parser": { "requireLimits": true }, + "storage": { + "requirePathValidation": true, + "requireCorruptionTests": true, + "requireReleaseEvidence": true + }, + "releaseEvidence": { + "targetSoakProof": { + "requiredForProduction": true, + "proofFileEnv": "JERBOA_DB_TARGET_SOAK_PROOF_FILE", + "failClosedEnv": "JERBOA_DB_REQUIRE_TARGET_SOAK_PROOF", + "requiredMarkers": [ + "production_soak_status=current-run-recorded", + "cluster_status=release-host-sustained-recorded", + "transport_status=release-host-sustained-recorded", + "transport_authentication_status=release-host-authenticated-recorded", + "production_min_seconds_met=1" + ] + }, + "targetPeerHttpProof": { + "requiredForProduction": true, + "proofFileEnv": "JERBOA_DB_TARGET_PEER_HTTP_PROOF_FILE", + "failClosedEnv": "JERBOA_DB_REQUIRE_TARGET_PEER_HTTP_PROOF", + "requiredMarkers": [ + "peer_http_status=linux-release-host-recorded", + "epoll_native_status=linux-epoll-recorded", + "peer_http_auth_status=target-evidence-recorded" + ] + }, + "levelDbProof": { + "requiredForProduction": true, + "proofFileEnv": "JERBOA_DB_LEVELDB_PROOF_FILE", + "failClosedEnv": "JERBOA_DB_REQUIRE_LEVELDB_PROOF", + "requiredMarkers": [ + "leveldb_status=target-evidence-recorded", + "chez_leveldb_status=pinned-version-recorded", + "leveldb_native_status=pinned-version-recorded", + "leveldb_corruption_status=target-evidence-recorded" + ] + } + } }, "suppressions": [] } new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. --- a/Makefile +++ b/Makefile @@ -1,107 +1,215 @@ -JERBOA_DIR ?= $(HOME)/mine/jerboa +JERBOA_VERSION ?= v0.2.3 +JERBOA_TOOL_DIR ?= $(CURDIR)/.jerboa/bin +JERBUILD ?= $(if $(wildcard $(JERBOA_TOOL_DIR)/jerbuild),$(JERBOA_TOOL_DIR)/jerbuild,jerbuild) +JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) +ifeq ($(JH),) +JERBOA_DIR ?= $(realpath $(CURDIR)/../jerboa) +else +JERBOA_DIR ?= $(JH) +endif SCHEME ?= $(JERBOA_DIR)/.chez/bin/scheme -LIBDIRS = lib:$(JERBOA_DIR)/lib +LIBDIRS = $(CURDIR)/lib:$(JERBOA_DIR)/lib -# Chez external FFI libs (for LMDB, DuckDB, etc.) CHEZ_EXT_DIR ?= $(HOME)/src -CHEZ_EXT_LIBDIRS = $(CHEZ_EXT_DIR)/chez-lmdb:$(CHEZ_EXT_DIR)/chez-duckdb -FULL_LIBDIRS = $(LIBDIRS):$(CHEZ_EXT_LIBDIRS) - -.PHONY: test test-cluster test-transport test-transport-tls test-migrate test-peer build clean check bench bench-quick mbrainz mbrainz-quick showcase setup - -# One-time setup: create the libjerboa_native.so symlink that Jerboa's DuckDB -# loader expects. On macOS the upstream artifact is libjerboa_native.dylib -# but the loader looks for .so; the symlink bridges that gap. -setup: - @if [ ! -e lib/libjerboa_native.so ]; then \ - if [ -e $(JERBOA_DIR)/lib/libjerboa_native.dylib ]; then \ - ln -sf $(JERBOA_DIR)/lib/libjerboa_native.dylib lib/libjerboa_native.so; \ - echo "Symlinked lib/libjerboa_native.so -> $(JERBOA_DIR)/lib/libjerboa_native.dylib"; \ - elif [ -e $(JERBOA_DIR)/lib/libjerboa_native.so ]; then \ - ln -sf $(JERBOA_DIR)/lib/libjerboa_native.so lib/libjerboa_native.so; \ - echo "Symlinked lib/libjerboa_native.so -> $(JERBOA_DIR)/lib/libjerboa_native.so"; \ - else \ - echo "WARNING: libjerboa_native not found in $(JERBOA_DIR)/lib; DuckDB will be unavailable"; \ +CHEZ_EXT_LIBDIRS = $(CHEZ_EXT_DIR)/chez-lmdb:$(CHEZ_EXT_DIR)/chez-duckdb:$(CHEZ_EXT_DIR)/chez-leveldb +RUN_SS = JERBUILD="$(JERBUILD)" JERBOA_DIR="$(JERBOA_DIR)" SCHEME="$(SCHEME)" support/run-ss.sh + +DIST_DIR ?= dist/release-evidence +SBOM_DIR ?= dist/sbom +REPRO_DIR ?= dist/reproducibility +SOAK_DIR ?= dist/soak +CARGO_AUDIT ?= $(shell command -v cargo-audit 2>/dev/null || printf '%s/.cargo/bin/cargo-audit' "$$HOME") +LOCAL_JERBOA_DIR ?= $(CURDIR)/../jerboa +DEFAULT_JERBOA_NATIVE_MANIFEST = $(if $(wildcard $(LOCAL_JERBOA_DIR)/jerboa-native-rs/Cargo.toml),$(LOCAL_JERBOA_DIR)/jerboa-native-rs/Cargo.toml,$(JERBOA_DIR)/jerboa-native-rs/Cargo.toml) +JERBOA_NATIVE_MANIFEST ?= $(DEFAULT_JERBOA_NATIVE_MANIFEST) +JERBOA_NATIVE_CRATE_DIR = $(patsubst %/Cargo.toml,%,$(JERBOA_NATIVE_MANIFEST)) +JERBOA_NATIVE_TARGET_DIR ?= $(CURDIR)/.jerboa/native-target +JERBOA_NATIVE_FEATURES ?= duckdb_feat +JERBOA_NATIVE_NO_DEFAULT_FEATURES ?= 1 + +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Darwin) +NATIVE_EXT = dylib +else +NATIVE_EXT = so +endif +DEFAULT_JERBOA_NATIVE_LIB = $(if $(wildcard $(JERBOA_NATIVE_TARGET_DIR)/release/libjerboa_native.$(NATIVE_EXT)),$(JERBOA_NATIVE_TARGET_DIR)/release/libjerboa_native.$(NATIVE_EXT),$(if $(wildcard $(LOCAL_JERBOA_DIR)/jerboa-native-rs/target/release/libjerboa_native.$(NATIVE_EXT)),$(LOCAL_JERBOA_DIR)/jerboa-native-rs/target/release/libjerboa_native.$(NATIVE_EXT),$(JERBOA_DIR)/jerboa-native-rs/target/release/libjerboa_native.$(NATIVE_EXT))) +JERBOA_NATIVE_LIB ?= $(DEFAULT_JERBOA_NATIVE_LIB) +JERBOA_RELEASE_NATIVE_LIB ?= $(JERBOA_DIR)/lib/libjerboa_native.$(NATIVE_EXT) + +CORE_TESTS := tests/test-core.ss tests/test-migrate.ss + +.PHONY: help ensure-jerboa-tools ensure-jerboa-native setup test test-core test-migrate test-cluster test-transport test-transport-tls test-peer test-sql-translate test-leveldb test-lmdb build security audit verify sbom reproducibility-report soak-evidence release-evidence bench bench-quick mbrainz mbrainz-quick showcase clean +.DEFAULT_GOAL := help + +help: + @echo "jerboa-db" + @echo "" + @echo "Development:" + @echo " make ensure-jerboa-tools Install project-local Jerboa tools when needed" + @echo " make setup Build/link optional libjerboa_native into ./lib" + @echo " make test Run deterministic core and migration tests" + @echo " make build Import-check release modules" + @echo " make security Run local release security checks" + @echo " make audit Audit sibling Jerboa native crate when available" + @echo " make verify Run deterministic release gate" + @echo " make sbom Write SBOM/provenance evidence" + @echo " make reproducibility-report" + @echo " Compare repeated local gate evidence" + @echo " make soak-evidence Record cluster/backend/soak status" + @echo " make release-evidence Write release evidence under dist/" + @echo "" + @echo "Optional native/soak gates:" + @echo " make test-peer Requires Linux epoll-capable libjerboa_native" + @echo " make test-sql-translate Requires DuckDB-enabled libjerboa_native" + @echo " make test-leveldb Requires chez-leveldb and LevelDB native libs" + @echo " make test-cluster Multi-node Raft convergence; currently non-default" + @echo " make test-transport TCP Raft transport convergence; currently non-default" + +ensure-jerboa-tools: + @if [ -x "$(JERBUILD)" ] && "$(JERBUILD)" --jerboa-home >/dev/null 2>&1; then \ + echo "Jerboa toolchain available: $(JERBUILD)"; \ + elif command -v "$(JERBUILD)" >/dev/null 2>&1 && "$(JERBUILD)" --jerboa-home >/dev/null 2>&1; then \ + echo "Jerboa toolchain available: $(JERBUILD)"; \ + elif [ -x "$(SCHEME)" ]; then \ + echo "Using adjacent Jerboa checkout: $(JERBOA_DIR)"; \ + else \ + echo "=== Fetching Jerboa $(JERBOA_VERSION) release tools into $(JERBOA_TOOL_DIR) ==="; \ + sh support/ensure-jerboa.sh "$(JERBOA_VERSION)" "$(JERBOA_TOOL_DIR)"; \ + fi + +ensure-jerboa-native: + @if [ ! -f "$(JERBOA_NATIVE_MANIFEST)" ]; then \ + echo "WARNING: Jerboa native manifest not found: $(JERBOA_NATIVE_MANIFEST)"; \ + elif ! command -v cargo >/dev/null 2>&1; then \ + echo "WARNING: cargo not found; optional DuckDB/peer tests may be unavailable"; \ + else \ + flags=""; \ + if [ "$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)" = "1" ]; then \ + flags="$$flags --no-default-features"; \ + fi; \ + if [ -n "$(JERBOA_NATIVE_FEATURES)" ]; then \ + flags="$$flags --features $(JERBOA_NATIVE_FEATURES)"; \ fi; \ + echo "Building Jerboa native crate for DB evidence: features=$(JERBOA_NATIVE_FEATURES), no_default=$(JERBOA_NATIVE_NO_DEFAULT_FEATURES), target=$(JERBOA_NATIVE_TARGET_DIR)"; \ + cargo build --release --manifest-path "$(JERBOA_NATIVE_MANIFEST)" --target-dir "$(JERBOA_NATIVE_TARGET_DIR)" $$flags; \ + fi + +setup: ensure-jerboa-tools ensure-jerboa-native + @mkdir -p lib + @if [ -e "$(JERBOA_NATIVE_LIB)" ]; then \ + ln -sf "$(JERBOA_NATIVE_LIB)" lib/libjerboa_native.so; \ + echo "Symlinked lib/libjerboa_native.so -> $(JERBOA_NATIVE_LIB)"; \ + elif [ -e "$(JERBOA_RELEASE_NATIVE_LIB)" ]; then \ + ln -sf "$(JERBOA_RELEASE_NATIVE_LIB)" lib/libjerboa_native.so; \ + echo "Symlinked lib/libjerboa_native.so -> $(JERBOA_RELEASE_NATIVE_LIB)"; \ + else \ + echo "WARNING: libjerboa_native not found; optional DuckDB/peer tests may be unavailable"; \ + fi + +test: test-core test-migrate + +test-core: ensure-jerboa-tools + $(RUN_SS) tests/test-core.ss + +test-migrate: ensure-jerboa-tools + $(RUN_SS) tests/test-migrate.ss + +build: ensure-jerboa-tools + $(RUN_SS) support/import-check.ss + +security: + scripts/security-check.sh + +audit: + @echo "jerboa-db has no project-local Rust, Node, Python, or C package manager dependencies." + @if [ -f "$(JERBOA_NATIVE_MANIFEST)" ] && [ -x "$(CARGO_AUDIT)" ]; then \ + echo "Auditing sibling Jerboa native crate: $(JERBOA_NATIVE_MANIFEST)"; \ + (cd "$(JERBOA_NATIVE_CRATE_DIR)" && "$(CARGO_AUDIT)" audit); \ + elif [ -f "$(JERBOA_NATIVE_MANIFEST)" ]; then \ + echo "WARNING: cargo-audit not found; sibling native audit skipped"; \ else \ - echo "lib/libjerboa_native.so already present"; \ + echo "WARNING: sibling Jerboa native manifest not found; native audit skipped"; \ fi -# Run the core test suite (in-memory, no FFI deps) -test: - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-core.ss - -# End-to-end Clojure-parity showcase (bookstore scenario). -# Exercises: Datalog, pull, as-of, pmap, atom+validator, STM+io!, -# agent+await-for, protocol, multi+hierarchy, spec, transducer, ->. -showcase: - $(SCHEME) --libdirs "$(LIBDIRS)" --script examples/bookstore.ss - -# Run cluster (Raft replication) tests -test-cluster: - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-cluster.ss - -# Run TCP transport tests (two nodes connected via loopback) -test-transport: - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-transport.ss - -# Run TLS transport tests (requires libssl + self-signed cert). -# Generate the cert with: -# mkdir -p $(JERBOA_DB_TLS_DIR) && \ -# openssl req -new -x509 -nodes -newkey rsa:2048 \ -# -keyout $(JERBOA_DB_TLS_DIR)/server.key \ -# -out $(JERBOA_DB_TLS_DIR)/server.crt \ -# -days 1 -subj "/CN=localhost" -JERBOA_DB_TLS_DIR ?= $(PREFIX)/tmp/jerboa-db-tls -test-transport-tls: - JERBOA_DB_TLS_DIR=$(JERBOA_DB_TLS_DIR) \ - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-transport-tls.ss - -# Run schema migration tests (rename/retype/delete/merge/split) -test-migrate: - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-migrate.ss - -# Run peer client tests (HTTP server + remote-entity + named DB + cache) -test-peer: - $(SCHEME) --libdirs "$(LIBDIRS)" --script tests/test-peer.ss - -# Run tests including LMDB backend -test-lmdb: - $(SCHEME) --libdirs "$(FULL_LIBDIRS)" --script tests/test-lmdb.ss - -# Compile all libraries (catches syntax/import errors) -build: - @echo "Compiling jerboa-db libraries..." - printf '(import (jerboa-db core))\n' | $(SCHEME) --libdirs "$(LIBDIRS)" - @echo "Build check complete." - -# Syntax check all .ss files -check: - @echo "Checking library files..." - @for f in $$(find lib -name "*.ss"); do \ - echo " $$f"; \ - $(SCHEME) --libdirs "$(LIBDIRS)" --script /dev/null 2>&1 | head -5 || true; \ - done - @echo "Check complete." - -# Full load test (all 7 scenarios at full scale) -bench: - $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/load-test.ss - -# Quick load test (1/10 scale — runs in under 5s) -bench-quick: - $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/load-test.ss --quick - -# MBrainz benchmark (8 standard queries, synthetic data at full scale) -mbrainz: - $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/mbrainz-bench.ss - -# MBrainz quick smoke test (1% scale, ~3 runs per query) -mbrainz-quick: - $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/mbrainz-bench.ss --quick - -# Clean compiled artifacts +verify: security build test audit + +sbom: ensure-jerboa-native + JERBUILD="$(JERBUILD)" JERBOA_DIR="$(JERBOA_DIR)" JERBOA_DB_SBOM_DIR="$(SBOM_DIR)" JERBOA_NATIVE_MANIFEST="$(JERBOA_NATIVE_MANIFEST)" JERBOA_NATIVE_TARGET_DIR="$(JERBOA_NATIVE_TARGET_DIR)" JERBOA_NATIVE_FEATURES="$(JERBOA_NATIVE_FEATURES)" JERBOA_NATIVE_NO_DEFAULT_FEATURES="$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)" JERBOA_NATIVE_LIB="$(JERBOA_NATIVE_LIB)" scripts/sbom.sh + +reproducibility-report: + JERBUILD="$(JERBUILD)" JERBOA_DIR="$(JERBOA_DIR)" JERBOA_DB_REPRO_DIR="$(REPRO_DIR)" scripts/reproducibility-report.sh + +soak-evidence: + JERBUILD="$(JERBUILD)" JERBOA_DIR="$(JERBOA_DIR)" JERBOA_DB_SOAK_DIR="$(SOAK_DIR)" scripts/soak-evidence.sh + +release-evidence: verify reproducibility-report sbom soak-evidence + rm -rf "$(DIST_DIR)" + mkdir -p "$(DIST_DIR)" + git rev-parse HEAD > "$(DIST_DIR)/git-commit.txt" + git status --short > "$(DIST_DIR)/git-status.txt" + uname -a > "$(DIST_DIR)/build-environment.txt" + @printf 'JERBOA_DIR=%s\nSCHEME=%s\nJERBOA_NATIVE_TARGET_DIR=%s\nJERBOA_NATIVE_FEATURES=%s\nJERBOA_NATIVE_NO_DEFAULT_FEATURES=%s\nJERBOA_NATIVE_LIB=%s\n' "$(JERBOA_DIR)" "$(SCHEME)" "$(JERBOA_NATIVE_TARGET_DIR)" "$(JERBOA_NATIVE_FEATURES)" "$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)" "$(JERBOA_NATIVE_LIB)" >> "$(DIST_DIR)/build-environment.txt" + @if [ -x "$(SCHEME)" ]; then \ + "$(SCHEME)" --version >> "$(DIST_DIR)/build-environment.txt" 2>&1; \ + else \ + printf 'SCHEME_STATUS=unavailable-cache-uses-jerbuild-exec\n' >> "$(DIST_DIR)/build-environment.txt"; \ + fi + @$(JERBUILD) --version >> "$(DIST_DIR)/build-environment.txt" 2>&1 || true + @$(JERBUILD) --jerboa-home >> "$(DIST_DIR)/build-environment.txt" 2>&1 || true + @$(MAKE) security > "$(DIST_DIR)/security-check.txt" + @$(MAKE) build > "$(DIST_DIR)/import-check.txt" + @$(MAKE) test > "$(DIST_DIR)/test.txt" + @$(MAKE) audit > "$(DIST_DIR)/dependency-audit.txt" 2>&1 + @if [ -f "$(JERBOA_NATIVE_MANIFEST)" ]; then \ + cargo metadata --manifest-path "$(JERBOA_NATIVE_MANIFEST)" --locked --format-version 1 > "$(DIST_DIR)/cargo-metadata-jerboa-native.json" 2>/dev/null || true; \ + fi + find lib bin tests benchmarks examples support -type f -name '*.ss' -print | sort | xargs shasum -a 256 > "$(DIST_DIR)/source-sha256.txt" + shasum -a 256 Makefile .jerboa/security.json SECURITY.md README.md docs/threat-model.md docs/storage-hardening.md docs/release-evidence.md scripts/security-check.sh scripts/sbom.sh scripts/reproducibility-report.sh scripts/soak-evidence.sh support/import-check.ss support/run-ss.sh > "$(DIST_DIR)/release-inputs-sha256.txt" + cp -R "$(SBOM_DIR)" "$(DIST_DIR)/sbom" + cp -R "$(REPRO_DIR)" "$(DIST_DIR)/reproducibility" + cp -R "$(SOAK_DIR)" "$(DIST_DIR)/soak" + +showcase: ensure-jerboa-tools + $(RUN_SS) examples/bookstore.ss + +test-cluster: ensure-jerboa-tools + $(RUN_SS) tests/test-cluster.ss + +test-transport: ensure-jerboa-tools + $(RUN_SS) tests/test-transport.ss + +test-transport-tls: ensure-jerboa-tools + JERBOA_DB_TLS_DIR=$${JERBOA_DB_TLS_DIR:-$${PREFIX:-/tmp}/tmp/jerboa-db-tls} \ + $(RUN_SS) tests/test-transport-tls.ss + +test-peer: setup + $(RUN_SS) tests/test-peer.ss + +test-sql-translate: setup + $(RUN_SS) tests/test-sql-translate.ss + +test-leveldb: setup + EXTRA_LIBDIRS="$(CHEZ_EXT_LIBDIRS)" $(RUN_SS) tests/test-leveldb-index.ss + +test-lmdb: setup + EXTRA_LIBDIRS="$(CHEZ_EXT_LIBDIRS)" $(RUN_SS) tests/test-lmdb.ss + +bench: ensure-jerboa-tools + $(RUN_SS) benchmarks/load-test.ss + +bench-quick: ensure-jerboa-tools + $(RUN_SS) benchmarks/load-test.ss --quick + +mbrainz: ensure-jerboa-tools + $(RUN_SS) benchmarks/mbrainz-bench.ss + +mbrainz-quick: ensure-jerboa-tools + $(RUN_SS) benchmarks/mbrainz-bench.ss --quick + clean: - find lib -name "*.so" -delete - find lib -name "*.wpo" -delete - rm -rf /tmp/jerboa-db-test-* + find lib -name "*.so" -delete 2>/dev/null || true + find lib -name "*.wpo" -delete 2>/dev/null || true + rm -rf dist + rm -rf .jerboa/native-target + rm -rf /tmp/jerboa-db-test-* /tmp/jdb-segtree-test /tmp/jdb-segtree-corrupt-test /tmp/jdb-persist-demo --- a/README.md +++ b/README.md @@ -11,11 +11,31 @@ About 10K lines of Scheme. ```bash make setup # one-time toolchain prep on macOS -make test # 37 core tests +make test # deterministic core + migration tests +make verify # security, import, test, and native-audit gate +make release-evidence # writes dist/release-evidence/ make mbrainz-quick # 1% MBrainz benchmark, < 5s make mbrainz # full bench (slow on a laptop) ``` +## Security status + +`jerboa-db` is not public-production-ready yet. The deterministic local gate is +`make verify`; it covers security checks, import checks, storage/corruption +regression tests, migrations, and a sibling Jerboa native RustSec audit when +available. + +The storage layer now validates segment-store paths, fixed segment filenames, +content-addressed segment reads, persisted roots, backups, transaction-log +segment names, and transport frame sizes. See [`SECURITY.md`](SECURITY.md), +[`docs/threat-model.md`](docs/threat-model.md), and +[`docs/storage-hardening.md`](docs/storage-hardening.md). + +The remaining release blockers are Linux peer tests with epoll-capable native +support, DuckDB analytics tests with a DuckDB-enabled native crate, stable +multi-node cluster/transport soak, LevelDB backend evidence, SBOM/reproducible +build notes, history secret scan, and external review. + ## What it does - **Datalog** — `(q '[:find ?e :where [?e :artist/name ?n]] db)` new file mode 100644 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,53 @@ +# Security Policy + +`jerboa-db` is a storage and query engine. Treat it as private release-candidate +software until the release gates in `docs/release-evidence.md` are complete. + +## Supported Status + +No public production-support commitment exists yet. A security-sensitive release +must be cut from a clean checkout after: + +- `make verify` passes. +- `make release-evidence` completes and its artifacts are reviewed. +- `jerboa_security_scan` reports no medium-or-higher findings. +- Native dependency evidence is recorded for the exact Jerboa toolchain used. +- Linux peer/transport and DuckDB/LevelDB optional gates are either passing or + explicitly excluded from the release scope. + +## Security Model + +- Query and transaction forms are trusted application inputs. Do not expose the + REPL, EDN import, or raw query endpoints directly to anonymous users. +- Persistent segment stores validate directory paths, segment IDs, segment + filenames, content hashes, root shapes, and segment decoder limits. +- Backups and transaction logs are local storage inputs. Restore/replay validates + headers and entry shapes before applying data. +- Peer HTTP and Raft transport are trusted-cluster surfaces. Use controlled + networks or TLS, and do not treat FASL transport frames as an Internet-safe + public protocol. +- DuckDB, LevelDB, epoll, TLS, and other native behavior comes from the selected + Jerboa toolchain. Record Cargo metadata and RustSec results before release. + +## Required Local Gate + +```sh +make verify +make release-evidence +``` + +`make verify` runs local security checks, import checks, core storage tests, +migration tests, and a sibling Jerboa native RustSec audit when available. + +The following gates remain release blockers for scopes that need them: + +- `make test-peer` on Linux with epoll-capable `libjerboa_native`. +- `make test-sql-translate` with DuckDB-enabled `libjerboa_native`. +- `make test-cluster` and `make test-transport` with stable convergence/teardown. +- LevelDB backend tests with pinned native dependency evidence. + +## Reporting + +Before public release, report issues privately to the repository owner. After +public release, replace this section with a dedicated advisory contact, +supported versions, and disclosure window. --- a/benchmarks/segtree-persist-read.ss +++ b/benchmarks/segtree-persist-read.ss @@ -1,7 +1,13 @@ -(import (chezscheme) (jerboa-db datom) (jerboa-db index segtree)) +(import (scheme) + (only (std security taint) check-untainted!) + (jerboa-db datom) + (jerboa-db index segtree)) (define dir "/tmp/jdb-persist-demo") +(define root-path (string-append dir "/root.bin")) +(check-untainted! dir 'segtree-persist-read) +(check-untainted! root-path 'segtree-persist-read) (define ss (make-fs-segstore dir)) -(define rb (call-with-port (open-file-input-port (string-append dir "/root.bin")) (lambda (p) (get-bytevector-all p)))) +(define rb (call-with-port (open-file-input-port root-path) (lambda (p) (get-bytevector-all p)))) (define t (segtree-load rb compare-datoms-eavt)) (define hit (segtree-range->list t ss (make-datom 500 0 +min-val+ 0 #t) (make-datom 500 (greatest-fixnum) +max-val+ (greatest-fixnum) #t))) --- a/benchmarks/segtree-persist-write.ss +++ b/benchmarks/segtree-persist-write.ss @@ -1,9 +1,16 @@ -(import (chezscheme) (jerboa-db datom) (jerboa-db index segtree)) +(import (scheme) + (only (std security taint) check-untainted! safe-delete-file) + (jerboa-db datom) + (jerboa-db index segtree)) (define dir "/tmp/jdb-persist-demo") -(when (file-exists? dir) (for-each (lambda (f) (delete-file (string-append dir "/" f))) (directory-list dir))) +(define root-path (string-append dir "/root.bin")) +(check-untainted! dir 'segtree-persist-write) +(check-untainted! root-path 'segtree-persist-write) +(when (file-exists? dir) + (for-each (lambda (f) (safe-delete-file (string-append dir "/" f))) (directory-list dir))) (define ss (make-fs-segstore dir)) (define ds (let loop ([i 0] [a '()]) (if (= i 1000) a (loop (+ i 1) (cons (make-datom i 1 (* i 10) 1 #t) a))))) (define t (segtree-build ss ds compare-datoms-eavt 64)) -(call-with-port (open-file-output-port (string-append dir "/root.bin")) +(call-with-port (open-file-output-port root-path) (lambda (p) (put-bytevector p (segtree-save t)))) (printf "WROTE ~a datoms -> ~a leaf files on disk~n" (segtree-total t) (segstore-size ss)) new file mode 100644 --- /dev/null +++ b/docs/release-evidence.md @@ -0,0 +1,92 @@ +# Release Evidence + +Run: + +```sh +make verify +make release-evidence +``` + +The evidence directory is `dist/release-evidence/` and is intentionally ignored. +It records: + +- Git commit and working-tree status. +- Build environment and selected Jerboa toolchain. +- Core/migration/import/security verification output. +- Optional sibling Jerboa native Cargo metadata and RustSec output when present. +- Repo-local DuckDB native-library feature posture and selected native-library + hash when present. +- SHA-256 hashes for release inputs and Scheme source files. +- `sbom/` with source, release-input, sibling-native metadata, and + no-project-local-package-manager dependency provenance. +- `reproducibility/` with two repeated local security/import/test passes and + `result.txt`. +- `soak/` with local cluster smoke status, bounded TCP transport smoke status, + authenticated frame smoke status, DuckDB/LevelDB backend smoke status, and + explicit blocked status for target-host peer HTTP and sustained load gates. + If a local smoke is flaky, missing a required native dependency, or hangs, + the evidence records the failed, missing-dependency, or timed-out status + instead of hiding the failure. + +Required release markers: + +- `transport_timeout_status: documented` +- `transport_timeout_seconds_default: 60` +- `transport_authentication_status: documented` +- `transport_authentication_smoke_status: local-smoke-recorded` +- `native_target_isolation_status: repo-local` +- `duckdb_native_feature_status: no-default-duckdb_feat` +- `duckdb_backend_status: optional-gated` +- `leveldb_backend_status: optional-gated` +- `production_soak_status: release-host-required` +- `target_soak_proof_status: documented` +- `target_peer_http_proof_status: documented` +- `leveldb_proof_status: documented` +- `proof_failure_status: blocked-target-proof` + +`make verify` is the deterministic local gate. The following production gates are +tracked but not part of the default Darwin-local gate: + +- Sustained multi-node cluster and transport convergence under release-host + load. The local release gate records short in-process cluster smoke plus a + bounded TCP transport smoke with an outer timeout. +- `make test-transport` stable TCP transport convergence, authenticated frame + tamper-rejection smoke, and teardown without hitting the bounded evidence + timeout. +- `make test-peer` on Linux with epoll-capable `libjerboa_native`. +- `make test-sql-translate` with DuckDB native symbols available; soak + evidence builds the selected sibling `jerboa-native-rs` crate into + `.jerboa/native-target` with Rust default features disabled and only + `duckdb_feat` enabled. It records `duckdb_status=local-smoke-recorded` when + the smoke passes and `duckdb_status=blocked-native-feature-missing` when the + selected native library or tooling is unavailable. +- `make test-leveldb` with `chez-leveldb`, `leveldb_shim`, and LevelDB native + libraries available; soak evidence records `leveldb_status` explicitly. + +## Target Proof Controls + +Release CI can require target-host evidence by setting the matching +`JERBOA_DB_REQUIRE_*_PROOF=1` variable. Missing or marker-incomplete proof files +fail `make soak-evidence` with `status=blocked-target-proof`; valid proofs are +copied into `dist/release-evidence/soak/` with `.sha256` sidecars. + +- `JERBOA_DB_TARGET_SOAK_PROOF_FILE` with + `JERBOA_DB_REQUIRE_TARGET_SOAK_PROOF=1` records sustained release-host + transport/cluster evidence. The proof must contain + `production_soak_status=current-run-recorded`, + `cluster_status=release-host-sustained-recorded`, + `transport_status=release-host-sustained-recorded`, + `transport_authentication_status=release-host-authenticated-recorded`, and + `production_min_seconds_met=1`. +- `JERBOA_DB_TARGET_PEER_HTTP_PROOF_FILE` with + `JERBOA_DB_REQUIRE_TARGET_PEER_HTTP_PROOF=1` records Linux peer HTTP evidence + with epoll-capable native support. The proof must contain + `peer_http_status=linux-release-host-recorded`, + `epoll_native_status=linux-epoll-recorded`, and + `peer_http_auth_status=target-evidence-recorded`. +- `JERBOA_DB_LEVELDB_PROOF_FILE` with + `JERBOA_DB_REQUIRE_LEVELDB_PROOF=1` records LevelDB backend evidence. The + proof must contain `leveldb_status=target-evidence-recorded`, + `chez_leveldb_status=pinned-version-recorded`, + `leveldb_native_status=pinned-version-recorded`, and + `leveldb_corruption_status=target-evidence-recorded`. new file mode 100644 --- /dev/null +++ b/docs/storage-hardening.md @@ -0,0 +1,66 @@ +# Storage Hardening + +This repository stores data in several formats: in-memory indices, persistent +segment-tree leaves, saved segment-tree roots, backup files, and transaction-log +segments. + +## Segment Tree Store + +- `make-fs-segstore` accepts only non-empty, NUL-free directory strings up to + 4096 characters. +- Segment IDs must be 8-byte bytevectors. +- Segment filenames must be exactly `<16 lowercase hex chars>.seg`. +- Segment writes go through a temp file and `rename-file`, so crashes do not + intentionally publish partial segment files. +- Segment reads recompute `content-hash-bytes` and reject mismatches before + decoding. +- Segment decoding rejects unsupported versions, oversized segment bytevectors, + oversized row counts, invalid value column types, truncated payloads, oversized + mixed-value payloads, and malformed embedded FASL values. + +The content hash is FNV-1a, so it is a corruption guard, not an adversarial +tamper-proof MAC. Hostile storage requires filesystem integrity, signatures, or +authenticated encryption above this layer. + +## Roots, Backups, And Logs + +- `segtree-load` validates root payload size, header shape, entry shape, segment + ID shape, counts, and datom tuple shape before constructing the tree. +- `restore!` validates backup magic, compression flags, payload shape, and datom + entry shape before replay. +- Transaction-log directories and segment filenames are checked before append or + replay. Segment names must be `segment-N.fasl`. + +## Optional Native Stores + +DuckDB, LevelDB, peer HTTP, and Linux epoll support depend on selected native +tooling. DuckDB evidence uses the selected sibling `jerboa-native-rs` crate, but +builds it into this repository's `.jerboa/native-target` with Rust default +features disabled and only `duckdb_feat` enabled, so unrelated native features +from another checkout or cache cannot accidentally satisfy the DB gate. LevelDB +still depends on external Chez/LevelDB bindings. Full production evidence must +also include Linux peer tests and sustained release-host transport/cluster load. + +Transport evidence markers: + +- `transport_timeout_status: documented` +- `transport_timeout_seconds_default: 60` +- `transport_authentication_status: documented` +- `transport_authentication_smoke_status: local-smoke-recorded` +- `native_target_isolation_status: repo-local` +- `duckdb_native_feature_status: no-default-duckdb_feat` +- `duckdb_backend_status: optional-gated` +- `leveldb_backend_status: optional-gated` +- `production_soak_status: release-host-required` +- `target_soak_proof_status: documented` +- `target_peer_http_proof_status: documented` +- `leveldb_proof_status: documented` +- `proof_failure_status: blocked-target-proof` + +Target-host release packets can satisfy the remaining non-Darwin gates through +fail-closed proof files. Sustained transport/cluster proofs must record +release-host duration, authenticated transport status, and successful cluster +convergence. Linux peer HTTP proofs must identify the epoll-capable native +library and peer authentication result. LevelDB proofs must pin both +`chez-leveldb` and native LevelDB versions and include corruption-rejection +evidence. new file mode 100644 --- /dev/null +++ b/docs/threat-model.md @@ -0,0 +1,60 @@ +# jerboa-db Threat Model + +`jerboa-db` is a Datomic-shaped storage and query engine with optional peer, +transport, LevelDB, and DuckDB paths. Treat it as a private release candidate +until the gates in `docs/release-evidence.md` are complete. + +## Assets + +- Database datoms, transaction history, schema definitions, and fulltext terms. +- Persistent segment-tree leaves and saved roots. +- Backup files and transaction-log segments. +- Peer HTTP endpoints and Raft/transport messages. +- Optional native DuckDB/LevelDB state and shared Jerboa native libraries. + +## Trust Boundaries + +- Query and transaction forms are trusted application input, not public Internet + syntax. Do not expose `q`, import, or REPL commands directly to anonymous users. +- Backup, root, segment, and transaction-log files are local storage inputs. A + corrupted file must fail closed instead of being replayed silently. +- Transport FASL frames are trusted-cluster traffic only. Production peers must + be on a controlled network and should enable either transport HMAC + authentication, TLS with operator-managed identity policy, or both. HMAC mode + verifies frame authenticity before FASL decoding; it does not replace + release-host network policy, replay analysis, or external review. +- DuckDB, LevelDB, epoll, and TLS behavior comes from the selected Jerboa native + toolchain. Release evidence must record the exact toolchain and native audit. + +## Current Controls + +- Disk segment stores reject empty/NUL/oversized directory paths. +- Segment filenames are fixed content-addressed hex names. +- Segment writes use temp-file plus rename and remove temp files on failure. +- Segment reads recompute the content hash before decoding. +- Segment and root decoders enforce size/count/shape limits. +- Backup restore validates the magic header, compression flag, payload shape, + and datom entry shape before replay. +- Transaction-log directories and segment filenames are validated before use. +- Transport frames are capped at 8 MiB before allocation and FASL decode.