Harden storage, transport, and backup handling
ober
7ac399c43fd2e3f07ea5d9a02ff4386ebb72f7d0
--- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,18 +12,22 @@ permissions: env: JERBOA_VERSION: v0.2.3 JERBUILD: ${{ github.workspace }}/.jerboa/bin/jerbuild + # The current DB source needs reviewed Raft-wire and bounded-TLS APIs that + # have not yet shipped in v0.2.3. Keep hosted CI visibly blocked until a + # concrete containing release can replace this marker and version. + JERBOA_CORE_API_STATUS: blocked-pending-reviewed-release jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install system tools run: | set -eu sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential curl ca-certificates + sudo apt-get install -y --no-install-recommends build-essential curl ca-certificates openssl - name: Install cargo-audit run: cargo install cargo-audit --locked @@ -31,7 +35,17 @@ jobs: - name: Install Jerboa toolchain run: sh support/ensure-jerboa.sh "$JERBOA_VERSION" .jerboa/bin - - name: Verify + - name: Record Jerboa core compatibility status + run: | + { + echo "### Jerboa core compatibility" + echo + echo "Status: \`$JERBOA_CORE_API_STATUS\`" + echo + echo "The fail-closed API probe in make verify must reject v0.2.3 until a reviewed containing release is pinned." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Verify (includes fail-closed core API probe) run: make verify - name: Release evidence --- a/.github/workflows/security-baseline.yml +++ b/.github/workflows/security-baseline.yml @@ -13,7 +13,7 @@ jobs: baseline: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Required release files run: | --- a/Makefile +++ b/Makefile @@ -25,7 +25,9 @@ DEFAULT_JERBOA_NATIVE_MANIFEST = $(if $(wildcard $(LOCAL_JERBOA_DIR)/jerboa-nati JERBOA_NATIVE_MANIFEST ?= $(DEFAULT_JERBOA_NATIVE_MANIFEST) JERBOA_NATIVE_CRATE_DIR = $(patsubst %/Cargo.toml,%,$(JERBOA_NATIVE_MANIFEST)) JERBOA_NATIVE_TARGET_DIR ?= $(CURDIR)/.jerboa/native-target -JERBOA_NATIVE_FEATURES ?= duckdb_feat +# fiber-httpd/fiber-ws uses the native SHA-1 helper during WebSocket setup; +# peer tests therefore require crypto as well as the optional DuckDB backend. +JERBOA_NATIVE_FEATURES ?= duckdb_feat,crypto JERBOA_NATIVE_NO_DEFAULT_FEATURES ?= 1 UNAME_S := $(shell uname -s) @@ -40,7 +42,7 @@ JERBOA_RELEASE_NATIVE_LIB ?= $(JERBOA_DIR)/lib/libjerboa_native.$(NATIVE_EXT) CORE_TESTS := tests/test-core.ss tests/test-migrate.ss -.PHONY: help ensure-jerboa-tools ensure-jerboa-native setup test test-core test-migrate test-cluster test-transport test-transport-tls test-peer test-sql-translate test-leveldb test-lmdb build security audit verify sbom reproducibility-report soak-evidence release-evidence bench bench-quick mbrainz mbrainz-quick showcase clean +.PHONY: help ensure-jerboa-tools ensure-jerboa-core-api ensure-jerboa-native setup test test-core test-migrate test-backup-security test-server-security test-cluster test-transport test-transport-tls test-transport-tls-admission test-transport-security test-peer test-sql-translate test-leveldb test-lmdb build security audit verify sbom reproducibility-report soak-evidence release-evidence bench bench-quick mbrainz mbrainz-quick showcase clean .DEFAULT_GOAL := help help: @@ -49,7 +51,7 @@ help: @echo "Development:" @echo " make ensure-jerboa-tools Install project-local Jerboa tools when needed" @echo " make setup Build/link optional libjerboa_native into ./lib" - @echo " make test Run deterministic core and migration tests" + @echo " make test Run core/storage and required transport security tests" @echo " make build Import-check release modules" @echo " make security Run local release security checks" @echo " make audit Audit sibling Jerboa native crate when available" @@ -65,7 +67,9 @@ help: @echo " make test-sql-translate Requires DuckDB-enabled libjerboa_native" @echo " make test-leveldb Requires chez-leveldb and LevelDB native libs" @echo " make test-cluster Multi-node Raft convergence; currently non-default" - @echo " make test-transport TCP Raft transport convergence; currently non-default" + @echo " make test-transport TCP Raft transport convergence" + @echo " make test-transport-security" + @echo " Required plain + generated-fixture TLS security gate" ensure-jerboa-tools: @if [ -x "$(JERBUILD)" ] && "$(JERBUILD)" --jerboa-home >/dev/null 2>&1; then \ @@ -79,6 +83,13 @@ ensure-jerboa-tools: sh support/ensure-jerboa.sh "$(JERBOA_VERSION)" "$(JERBOA_TOOL_DIR)"; \ fi +ensure-jerboa-core-api: ensure-jerboa-tools + @if ! $(RUN_SS) support/core-api-check.ss; then \ + echo "ERROR: selected Jerboa toolchain lacks the reviewed Raft wire and bounded TLS APIs required by this source." >&2; \ + echo "Hosted CI is intentionally blocked until a Jerboa release newer than v0.2.3 contains those APIs; do not skip the transport security gate." >&2; \ + exit 1; \ + fi + ensure-jerboa-native: @if [ ! -f "$(JERBOA_NATIVE_MANIFEST)" ]; then \ echo "WARNING: Jerboa native manifest not found: $(JERBOA_NATIVE_MANIFEST)"; \ @@ -108,15 +119,26 @@ setup: ensure-jerboa-tools ensure-jerboa-native echo "WARNING: libjerboa_native not found; optional DuckDB/peer tests may be unavailable"; \ fi -test: test-core test-migrate +test: test-core test-migrate test-backup-security test-server-security test-transport-security -test-core: ensure-jerboa-tools +test-core: ensure-jerboa-core-api $(RUN_SS) tests/test-core.ss -test-migrate: ensure-jerboa-tools +test-migrate: ensure-jerboa-core-api $(RUN_SS) tests/test-migrate.ss -build: ensure-jerboa-tools +test-backup-security: ensure-jerboa-core-api + $(RUN_SS) tests/test-backup-security.ss + +ifeq ($(UNAME_S),Darwin) +test-server-security: ensure-jerboa-core-api + @echo "SKIP: fiber-httpd uses Linux epoll; run test-server-security on Linux release host" +else +test-server-security: ensure-jerboa-core-api setup + $(RUN_SS) tests/test-server-security.ss +endif + +build: ensure-jerboa-core-api $(RUN_SS) support/import-check.ss security: @@ -133,7 +155,7 @@ audit: echo "WARNING: sibling Jerboa native manifest not found; native audit skipped"; \ fi -verify: security build test audit +verify: ensure-jerboa-core-api security build test audit sbom: ensure-jerboa-native JERBUILD="$(JERBUILD)" JERBOA_DIR="$(JERBOA_DIR)" JERBOA_DB_SBOM_DIR="$(SBOM_DIR)" JERBOA_NATIVE_MANIFEST="$(JERBOA_NATIVE_MANIFEST)" JERBOA_NATIVE_TARGET_DIR="$(JERBOA_NATIVE_TARGET_DIR)" JERBOA_NATIVE_FEATURES="$(JERBOA_NATIVE_FEATURES)" JERBOA_NATIVE_NO_DEFAULT_FEATURES="$(JERBOA_NATIVE_NO_DEFAULT_FEATURES)" JERBOA_NATIVE_LIB="$(JERBOA_NATIVE_LIB)" scripts/sbom.sh @@ -172,7 +194,7 @@ release-evidence: verify reproducibility-report sbom soak-evidence cargo metadata --manifest-path "$(JERBOA_NATIVE_MANIFEST)" --locked --format-version 1 > "$(DIST_DIR)/cargo-metadata-jerboa-native.json" 2>/dev/null || true; \ fi find lib bin tests benchmarks examples support -type f -name '*.ss' -print | sort | xargs shasum -a 256 > "$(DIST_DIR)/source-sha256.txt" - shasum -a 256 Makefile .jerboa/security.json SECURITY.md README.md docs/threat-model.md docs/storage-hardening.md docs/release-evidence.md scripts/security-check.sh scripts/sbom.sh scripts/reproducibility-report.sh scripts/soak-evidence.sh scripts/sanitize-evidence.sh support/import-check.ss support/run-ss.sh > "$(DIST_DIR)/release-inputs-sha256.txt" + shasum -a 256 Makefile .jerboa/security.json SECURITY.md README.md docs/threat-model.md docs/storage-hardening.md docs/release-evidence.md scripts/security-check.sh scripts/sbom.sh scripts/reproducibility-report.sh scripts/soak-evidence.sh scripts/sanitize-evidence.sh support/core-api-check.ss support/import-check.ss support/run-ss.sh > "$(DIST_DIR)/release-inputs-sha256.txt" cp -R "$(SBOM_DIR)" "$(DIST_DIR)/sbom" cp -R "$(REPRO_DIR)" "$(DIST_DIR)/reproducibility" cp -R "$(SOAK_DIR)" "$(DIST_DIR)/soak" @@ -181,16 +203,36 @@ release-evidence: verify reproducibility-report sbom soak-evidence showcase: ensure-jerboa-tools $(RUN_SS) examples/bookstore.ss -test-cluster: ensure-jerboa-tools +test-cluster: ensure-jerboa-core-api $(RUN_SS) tests/test-cluster.ss -test-transport: ensure-jerboa-tools +test-transport: ensure-jerboa-core-api $(RUN_SS) tests/test-transport.ss -test-transport-tls: ensure-jerboa-tools +test-transport-tls: ensure-jerboa-core-api JERBOA_DB_TLS_DIR=$${JERBOA_DB_TLS_DIR:-$${PREFIX:-/tmp}/tmp/jerboa-db-tls} \ $(RUN_SS) tests/test-transport-tls.ss +test-transport-tls-admission: ensure-jerboa-core-api + JERBOA_DB_TLS_DIR=$${JERBOA_DB_TLS_DIR:-$${PREFIX:-/tmp}/tmp/jerboa-db-tls} \ + $(RUN_SS) tests/test-transport-tls-admission.ss + +test-transport-security: ensure-jerboa-core-api test-transport + @command -v openssl >/dev/null 2>&1 || { \ + echo "ERROR: openssl is required to generate the transport TLS regression fixture" >&2; \ + exit 1; \ + } + @set -eu; \ + umask 077; \ + tmpbase=$${TMPDIR:-/tmp}; \ + tlsdir=$$(mktemp -d "$$tmpbase/jerboa-db-tls.XXXXXX"); \ + trap 'rm -rf "$$tlsdir"' 0 HUP INT TERM; \ + openssl req -new -x509 -nodes -newkey rsa:2048 \ + -keyout "$$tlsdir/server.key" -out "$$tlsdir/server.crt" \ + -days 1 -subj "/CN=localhost" >/dev/null 2>&1; \ + JERBOA_DB_TLS_DIR="$$tlsdir" $(RUN_SS) tests/test-transport-tls.ss; \ + JERBOA_DB_TLS_DIR="$$tlsdir" $(RUN_SS) tests/test-transport-tls-admission.ss + test-peer: setup $(RUN_SS) tests/test-peer.ss --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ About 10K lines of Scheme. ```bash make setup # one-time toolchain prep on macOS -make test # deterministic core + migration tests +make test # core/storage + plain/TLS transport security tests make verify # security, import, test, and native-audit gate make release-evidence # writes dist/release-evidence/ make mbrainz-quick # 1% MBrainz benchmark, < 5s @@ -22,12 +22,18 @@ make mbrainz # full bench (slow on a laptop) `jerboa-db` is not public-production-ready yet. The deterministic local gate is `make verify`; it covers security checks, import checks, storage/corruption -regression tests, migrations, and a sibling Jerboa native RustSec audit when -available. +regressions, migrations, authenticated plain transport, generated-fixture TLS +replication/admission, and a sibling Jerboa native RustSec audit when available. +It first probes for the exact Raft wire and bounded TLS APIs this source uses. +The published Jerboa v0.2.3 toolchain does not contain them, so hosted CI is +intentionally fail-closed pending a reviewed containing Jerboa release. The storage layer now validates segment-store paths, fixed segment filenames, -content-addressed segment reads, persisted roots, backups, transaction-log -segment names, and transport frame sizes. See [`SECURITY.md`](SECURITY.md), +content-addressed segment reads, persisted roots, bounded/private backups, +transaction-log segment names, and transport frame sizes. The HTTP API binds +loopback only and requires a 32-byte-or-longer bearer token plus optional +per-database authorization; Raft defaults to loopback and rejects unauthenticated +public binding. See [`SECURITY.md`](SECURITY.md), [`docs/threat-model.md`](docs/threat-model.md), and [`docs/storage-hardening.md`](docs/storage-hardening.md). --- a/SECURITY.md +++ b/SECURITY.md @@ -21,11 +21,38 @@ must be cut from a clean checkout after: REPL, EDN import, or raw query endpoints directly to anonymous users. - Persistent segment stores validate directory paths, segment IDs, segment filenames, content hashes, root shapes, and segment decoder limits. -- Backups and transaction logs are local storage inputs. Restore/replay validates - headers and entry shapes before applying data. -- Peer HTTP and Raft transport are trusted-cluster surfaces. Use controlled - networks or TLS, and do not treat FASL transport frames as an Internet-safe - public protocol. +- Backups are created exclusively with mode `0600`; restore bounds compressed + and expanded bytes before decoding a versioned non-executable storage + envelope, then validates entry counts/shapes. Legacy native-FASL backups fail + closed. +- Local roots, mixed segment values, LevelDB datoms, and transaction-log records + use the same closed storage schema. It preserves pure data and supported Chez + time/typed-vector values, but rejects procedures and unsupported runtime + records/objects. Transaction logs use capped length-prefixed records, private + owner-held no-follow files, descriptor `fsync`, and per-log append locking. +- Peer HTTP is loopback-only, bearer-authenticated, connection-bounded, and can + enforce per-database authorization. Browser WebSocket origins are denied unless + explicitly allowlisted. Do not publish the cleartext port. +- Raft defaults to loopback. A non-loopback listener requires a 32-byte message + authentication key (TLS remains defense in depth). Authenticated startup also + requires explicit cluster and session/epoch IDs. Every versioned frame binds + those IDs plus sender and intended recipient; the receiver verifies the + sender is current cluster membership before Raft delivery. MAC verification + precedes bounded safe-data decoding. Only the four versioned Raft RPC wire + schemas are reconstructed; private log-entry records and local proposal + channels never cross the serialization boundary. Frames are limited to 1 MiB + and 100,000 decoded objects. Each authenticated sender uses a monotonic sequence; + receivers retain the highest sequence for every cluster/sender/recipient/ + session context, so an old frame cannot become valid after cache pressure. + Session IDs must be unique for each sender lifetime and rotated on sender + restart or rekey. Inbound admission is capped, and one absolute pre-auth + deadline covers the TLS handshake plus the first authenticated frame, so + partial headers and silent TLS ClientHello connections cannot retain slots. + Established peer connections are serialized, lifetime-bounded, and registered + for immediate interruption during node shutdown; reconnect backoff observes + the node stop signal. + Treat this as a private cluster protocol and rotate the key and session ID + together during rekeying. - DuckDB, LevelDB, epoll, TLS, and other native behavior comes from the selected Jerboa toolchain. Record Cargo metadata and RustSec results before release. @@ -36,14 +63,20 @@ make verify make release-evidence ``` -`make verify` runs local security checks, import checks, core storage tests, -migration tests, and a sibling Jerboa native RustSec audit when available. +`make verify` runs a fail-closed Jerboa core API compatibility probe, local +security/import checks, core storage and migration tests, authenticated plain +transport convergence, and generated-fixture TLS replication/admission tests. +It also runs a sibling Jerboa native RustSec audit when available. Jerboa v0.2.3 +lacks the required Raft wire and bounded TLS APIs; hosted CI remains explicitly +blocked until a reviewed containing release is pinned. The following gates remain release blockers for scopes that need them: - `make test-peer` on Linux with epoll-capable `libjerboa_native`. +- `make test-server-security` on Linux with epoll-capable `libjerboa_native`. - `make test-sql-translate` with DuckDB-enabled `libjerboa_native`. -- `make test-cluster` and `make test-transport` with stable convergence/teardown. +- Sustained release-host cluster/transport convergence and teardown soak (the + short plain/TLS security regressions are part of `make verify`). - LevelDB backend tests with pinned native dependency evidence. ## Reporting --- a/bin/jerboa-db.ss +++ b/bin/jerboa-db.ss @@ -2,7 +2,7 @@ ;;; jerboa-db CLI tool ;;; ;;; Usage: -;;; jerboa-db serve --port PORT --data PATH [--host HOST] +;;; jerboa-db serve --port PORT --data PATH --token-file PATH [--host HOST] ;;; jerboa-db stats --data PATH ;;; jerboa-db backup --data PATH --output PATH ;;; jerboa-db gc --data PATH [--all] @@ -13,7 +13,8 @@ (import (jerboa prelude) (except (jerboa-db core) log) (jerboa-db backup) - (jerboa-db gc)) + (jerboa-db gc) + (jerboa-db server)) ;; ---- Argument parsing ---- @@ -50,18 +51,20 @@ (def (cmd-serve opts) (let ([port (string->number (opt opts "port" "8484"))] [data (opt opts "data" ":memory:")] - [host (opt opts "host" "0.0.0.0")]) + [host (opt opts "host" "127.0.0.1")] + [token-file (opt opts "token-file" (getenv "JERBOA_DB_TOKEN_FILE"))]) + (unless token-file + (error 'serve "--token-file or JERBOA_DB_TOKEN_FILE is required")) (displayln "Starting Jerboa-DB server...") (displayln " Data: " data) (displayln " Bind: " host ":" port) - (let ([conn (connect data)]) - (guard (exn [#t (displayln "Error: server module unavailable: " - (condition-message exn))]) - (let ([server-mod (eval '(import (jerboa-db server)))]) - (eval `(start-server! ,conn ,host ,port)) - (displayln "Server running. Press Ctrl-C to stop.") - ;; Block forever - (let loop () (sleep (make-time 'time-duration 0 1)) (loop))))))) + (let* ([token (string-trim (read-file-string token-file))] + [conn (connect data)] + [server (start-server + (new-server-config conn port host token))]) + (displayln "Server running. Press Ctrl-C to stop.") + ;; Block forever. The process supervisor owns shutdown/cleanup. + (let loop () (sleep (make-time 'time-duration 0 1)) (loop))))) (def (cmd-stats opts) (let ([data (opt opts "data" ":memory:")]) --- a/docs/datomix-idea.md +++ b/docs/datomix-idea.md @@ -27,7 +27,7 @@ jerboa-db source. |---|------|-----------------|--------|--------| | 1 | Memory index = **B+-tree**, not red-black | RB-tree (`index/memory.ss`) | High (GC, scans, bulk load) | Med | | 2 | Index reads = **lazy cursors**, not lists | `dbi-range`/`dbi-seek` return lists | High (wide joins Q4/Q8) | Med | -| 3 | Durable segments = **columnar (transposed)** | per-datom FASL in LevelDB | High (analytics gap, compression) | High | +| 3 | Durable segments = **columnar (transposed)** | per-datom safe envelopes in LevelDB | High (analytics gap, compression) | High | | 4 | **Range cardinality from segment metadata** | `dbi-count` scans | High (planner quality) | Med | | 5 | **Immutable id-addressed segments + snapshots** | per-datom LevelDB put | High (snapshots, GC, as-of) | High | | 6 | **Byte-aware, sizing-based cache** | count-limited LRU | Med (predictable memory) | Low | @@ -139,8 +139,8 @@ structs — `eas` (entity+attr packed), `vs` (values), plus t/op columns, with **type-specialized primitive accessors** `getLongV` / `getIntV` / `getDoubleV` / `getFloatV` and `isAssertion(i)`. Homogeneous columns avoid boxing entirely. -**jerboa-db** — `index/leveldb.ss`: each datom is individually FASL-encoded -(`datom->fasl-bytevector`) and stored as its own LevelDB value. +**jerboa-db** — `index/leveldb.ss`: each datom is individually encoded with the +bounded, non-executable DB storage codec and stored as its own LevelDB value. **Why switch.** Column layout gives (a) strong compression — sorted runs of identical `e`/`a` and monotone `t` compress with delta/RLE; (b) fast scans over a @@ -151,8 +151,9 @@ owning it in the segment format shrinks that gap and feeds the planner. **Recommendation.** Introduce a segment as the unit of durable storage (item 5), and encode each segment column-transposed: parallel arrays for e, a, v (primitive-typed per attribute value-type), t, op; plus a small header (count, -min/max key, per-column codec). Store the whole segment as one value. Keep FASL -only for heterogeneous fallback columns. +min/max key, per-column codec). Store the whole segment as one value. Keep the +closed, length-prefixed storage codec for heterogeneous fallback columns; never +fall back to native FASL for replaceable storage. ## 4. Range cardinality straight from the directory · *high* · verified --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -7,12 +7,20 @@ make verify make release-evidence ``` +The gate begins with `support/core-api-check.ss`. It must select a Jerboa +toolchain containing the reviewed Raft wire adapters and bounded TLS +accept/deadline APIs. Published Jerboa v0.2.3 does not contain that closure, so +hosted CI records `blocked-pending-reviewed-release` and fails rather than +skipping transport security tests. A local workspace must use the exact current +sibling artifact until a reviewed containing release is pinned. + The evidence directory is `dist/release-evidence/` and is intentionally ignored. It records: - Git commit and working-tree status. - Build environment and selected Jerboa toolchain. -- Core/migration/import/security verification output. +- Core/migration/import/security verification output, including authenticated + plain transport plus generated-fixture TLS replication/admission. - Optional sibling Jerboa native Cargo metadata and RustSec output when present. - Repo-local DuckDB native-library feature posture and selected native-library hash when present. @@ -36,7 +44,7 @@ Required release markers: - `transport_authentication_status: documented` - `transport_authentication_smoke_status: local-smoke-recorded` - `native_target_isolation_status: repo-local` -- `duckdb_native_feature_status: no-default-duckdb_feat` +- `duckdb_native_feature_status: no-default-duckdb_feat-crypto` - `duckdb_backend_status: optional-gated` - `leveldb_backend_status: optional-gated` - `production_soak_status: release-host-required` @@ -53,20 +61,20 @@ Target proof files are rejected before copy when they are larger than `JERBOA_DB_TARGET_PROOF_MAX_BYTES` bytes or contain host-private paths, raw `git@` remotes, private-key blocks, or credential-shaped material. -`make verify` is the deterministic local gate. The following production gates are -tracked but not part of the default Darwin-local gate: +`make verify` is the deterministic local gate. It generates a private temporary +certificate/key fixture, runs `test-transport`, TLS multi-record replication, +pre-auth admission deadlines, and worker-zero shutdown, then removes the +fixture. The following production gates are tracked but not part of the default +Darwin-local gate: - Sustained multi-node cluster and transport convergence under release-host load. The local release gate records short in-process cluster smoke plus a bounded TCP transport smoke with an outer timeout. -- `make test-transport` stable TCP transport convergence, authenticated frame - tamper-rejection smoke, and teardown without hitting the bounded evidence - timeout. - `make test-peer` on Linux with epoll-capable `libjerboa_native`. - `make test-sql-translate` with DuckDB native symbols available; soak evidence builds the selected sibling `jerboa-native-rs` crate into `.jerboa/native-target` with Rust default features disabled and only - `duckdb_feat` enabled. It records `duckdb_status=local-smoke-recorded` when + `duckdb_feat,crypto` enabled. It records `duckdb_status=local-smoke-recorded` when the smoke passes and `duckdb_status=blocked-native-feature-missing` when the selected native library or tooling is unavailable. - `make test-leveldb` with `chez-leveldb`, `leveldb_shim`, and LevelDB native --- a/docs/storage-hardening.md +++ b/docs/storage-hardening.md @@ -16,7 +16,12 @@ segments. decoding. - Segment decoding rejects unsupported versions, oversized segment bytevectors, oversized row counts, invalid value column types, truncated payloads, oversized - mixed-value payloads, and malformed embedded FASL values. + mixed-value payloads, trailing bytes, and malformed embedded safe-data values. +- Mixed values, roots, LevelDB datoms, backups, and transaction-log records use + `(jerboa-db storage-codec)`, a closed tagged schema over `(std safe-fasl)`. + Pure pairs/vectors/bytevectors/scalars plus fxvectors, flvectors, and supported + Chez time values round-trip. Procedures, cycles, unsupported records, ports, + hashtables, conditions, and other runtime objects fail closed. The content hash is FNV-1a, so it is a corruption guard, not an adversarial tamper-proof MAC. Hostile storage requires filesystem integrity, signatures, or @@ -29,15 +34,41 @@ authenticated encryption above this layer. - `restore!` validates backup magic, compression flags, payload shape, and datom entry shape before replay. - Transaction-log directories and segment filenames are checked before append or - replay. Segment names must be `segment-N.fasl`. + replay. Segment names remain `segment-N.fasl`, but the contents are `JDBT2000` + files with capped u32-length-prefixed safe records, not native FASL. Appends + use an owner-held, single-link regular descriptor opened with `O_NOFOLLOW`, + mode 0600, per-log locking, and descriptor `fsync`; truncated/oversized records + and trailing partial frames are errors. + +The storage format upgrade is deliberately fail closed: `JDBU1000` backups, +version-1 mixed segments, native-FASL roots/LevelDB values, and pre-`JDBT2000` +transaction logs are rejected. Migrate trusted legacy data with an offline, +version-pinned converter in an isolated process; production readers never fall +back to native FASL. + +## Raft Transport + +Transport bodies use `(std safe-fasl)` with explicit 1 MiB and 100,000-object +limits. Before encoding, `(std raft)` maps its private log-entry records into a +versioned pure-data wire schema; after decoding, it validates one of the four +allowed RPC shapes before reconstructing records. Procedures, local reply +channels, unknown message tags, malformed fields, native FASL, and trailing data +fail closed. Authenticated frames verify HMAC, cluster/sender/recipient/session +identity, membership, and monotonic replay sequence before the inner Raft message +is decoded. Outbound peers reuse one serialized connection instead of performing +a TLS handshake for every heartbeat; failures close and reconnect with bounded, +stop-aware backoff. An admitted inbound connection can carry multiple frames, +has a 30-second maximum lifetime, and is interrupted immediately during node +shutdown. ## Optional Native Stores DuckDB, LevelDB, peer HTTP, and Linux epoll support depend on selected native tooling. DuckDB evidence uses the selected sibling `jerboa-native-rs` crate, but builds it into this repository's `.jerboa/native-target` with Rust default -features disabled and only `duckdb_feat` enabled, so unrelated native features -from another checkout or cache cannot accidentally satisfy the DB gate. LevelDB +features disabled and only `duckdb_feat,crypto` enabled (crypto supplies the +WebSocket SHA-1 boundary), so unrelated native features from another checkout +or cache cannot accidentally satisfy the DB gate. LevelDB still depends on external Chez/LevelDB bindings. Full production evidence must also include Linux peer tests and sustained release-host transport/cluster load. @@ -48,7 +79,7 @@ Transport evidence markers: - `transport_authentication_status: documented` - `transport_authentication_smoke_status: local-smoke-recorded` - `native_target_isolation_status: repo-local` -- `duckdb_native_feature_status: no-default-duckdb_feat` +- `duckdb_native_feature_status: no-default-duckdb_feat-crypto` - `duckdb_backend_status: optional-gated` - `leveldb_backend_status: optional-gated` - `production_soak_status: release-host-required` --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -18,10 +18,10 @@ until the gates in `docs/release-evidence.md` are complete. syntax. Do not expose `q`, import, or REPL commands directly to anonymous users. - Backup, root, segment, and transaction-log files are local storage inputs. A corrupted file must fail closed instead of being replayed silently. -- Transport FASL frames are trusted-cluster traffic only. Production peers must +- Transport safe-data frames are private-cluster traffic only. Production peers must be on a controlled network and should enable either transport HMAC authentication, TLS with operator-managed identity policy, or both. HMAC mode - verifies frame authenticity before FASL decoding; it does not replace + verifies frame authenticity before bounded safe-data decoding; it does not replace release-host network policy, replay analysis, or external review. - DuckDB, LevelDB, epoll, and TLS behavior comes from the selected Jerboa native toolchain. Release evidence must record the exact toolchain and native audit. @@ -33,24 +33,36 @@ until the gates in `docs/release-evidence.md` are complete. - Segment writes use temp-file plus rename and remove temp files on failure. - Segment reads recompute the content hash before decoding. - Segment and root decoders enforce size/count/shape limits. -- Backup restore validates the magic header, compression flag, payload shape, - and datom entry shape before replay. -- Transaction-log directories and segment filenames are validated before use. -- Transport frames are capped at 8 MiB before allocation and FASL decode. -- Optional HMAC-SHA256 transport authentication validates `magic || tag || - payload` before any FASL payload is decoded, with keys required to be at - least 32 bytes. -- The default release gate runs core/migration tests, import checks, local - security checks, and a RustSec audit of the selected sibling Jerboa native +- Backup, root, mixed-segment, LevelDB, and transaction-log values use a closed, + bounded, non-executable storage envelope; native-FASL legacy files are + rejected before reconstruction. Backup restore additionally validates the + magic header, compression flag, payload shape, and datom entry shape. +- Transaction-log directories and segment filenames are validated before use; + append opens the held file with `O_NOFOLLOW`, checks regular/owner/single-link + metadata, forces mode 0600, frames each record with a capped length, fsyncs + the descriptor, and serializes concurrent writers on one log object. +- Transport frames are capped at 1 MiB and 100,000 decoded objects. The decoder + accepts only the versioned four-message Raft RPC schema and reconstructs private + log-entry records through `(std raft)` after safe-data validation. +- Optional HMAC-SHA256 transport authentication validates the tag over + `magic || nonce || payload` before any safe-data payload is decoded. Keys are + at least 32 bytes; the authenticated envelope also binds cluster, sender, + recipient, session, and a monotonic replay sequence. +- The default release gate first verifies the required Jerboa Raft/TLS API + closure, then runs core/migration/storage tests, import and local security + checks, authenticated plain transport convergence, generated-fixture TLS + replication/admission, and a RustSec audit of the selected sibling native crate when available. ## Remaining Release Blockers - Linux-hosted peer HTTP tests with epoll-capable `libjerboa_native`. - DuckDB analytics tests with a native crate built with `duckdb_feat` or `full`. -- Stable multi-node transport convergence tests and longer release-host soak - logs. Current local evidence covers short in-process cluster smoke, bounded - TCP transport smoke, and authenticated frame tamper-rejection smoke. +- Longer release-host multi-node transport/cluster soak logs. The deterministic + local gate covers short authenticated TCP convergence plus TLS replication, + admission deadlines, and worker-zero teardown. +- A reviewed Jerboa release containing the required Raft wire and bounded TLS + APIs; hosted CI is fail-closed while its pinned v0.2.3 lacks them. - LevelDB backend integration tests with documented native dependency versions. - History secret scan and external review. --- a/jerboa-db.md +++ b/jerboa-db.md @@ -117,7 +117,7 @@ they are unimplemented (❌), implemented (✅), or planned stubs (🚧). | HTTP server | ✅ Done | `server.ss` — all REST + WebSocket endpoints; CLI entrypoint 🚧 | | Remote peer client | ✅ Done | `peer.ss` — transact, query, pull, multi-URL failover | | Raft consensus / HA (in-process) | ✅ Done | `replication.ss` + `cluster.ss`; 6/6 tests | -| Raft consensus / HA (TCP transport) | ✅ Done | `transport.ss`; 12/12 tests; opt-in TLS via `:tls-config` argument (libssl-gated) | +| Raft consensus / HA (TCP transport) | ✅ Done | `transport.ss`; 23/23 plain/auth tests plus 7/7 TLS tests; bounded versioned safe-data wire schema | | Read replicas | ✅ Done | Follower apply fiber, ~50ms lag | ### Analytics @@ -867,10 +867,11 @@ fiber detects this case and treats the last log entry as effectively committed typically < 100 ms behind the leader). - **Transport (Phase 6.3, complete):** `transport.ss` bridges each node's - `(std raft)` channels to TCP sockets using length-framed FASL messages. - Outbound proxy fibers serialise and write; inbound accept-loop fibers - deserialise and deliver. Reconnect backoff (100ms → 5s) handles transient - failures. API: `start-transport-db-node!`, `transport-node-add-peer!`. + `(std raft)` channels to TCP sockets using bounded, length-framed safe-data messages. + Outbound proxy fibers serialise over one reusable connection per peer; + inbound workers deliver multiple frames per admitted connection. Six bounded + attempts with stop-aware 100ms–1.6s backoff handle transient failures. API: + `start-transport-db-node!`, `transport-node-add-peer!`. #### Peer caching (the real Datomic secret) @@ -2106,11 +2107,14 @@ never block the transactor. ```scheme (import (jerboa-db peer)) +(def token (string-trim (read-file-string "/run/secrets/jerboa-db-token"))) + ;; Single server: -(def conn (connect-remote "http://localhost:8484")) +(def conn (connect-remote "http://localhost:8484" #f token)) ;; Cluster with automatic failover across leader changes: -(def conn (connect-remote* '("http://node-0:8484" "http://node-1:8484"))) +(def conn (connect-remote* '("http://node-0:8484" "http://node-1:8484") + #f token)) ;; All basic operations work over HTTP: (remote-transact! conn tx-ops) ;; → tx-report alist @@ -2149,9 +2153,9 @@ Note: operations are server-side round-trips. Local db-value caching multiple read replicas (followers). **Current state:** Complete. Both the in-process layer (6/6 tests) and the TCP -transport layer (12/12 tests) are implemented and passing. Each `start-transport-db-node!` +transport layer (23/23 plain/auth tests plus 7/7 TLS tests) are implemented. Each `start-transport-db-node!` call returns a `(values transport-node replicated-conn)` pair compatible with the -full cluster API. The remaining items are TLS wrapping for production use and +full cluster API. TLS wrapping is implemented; the remaining items are local db-value caching in the peer client (Phase 6.4). #### 6.1 Raft-Based Transactor ✅ Complete (in-process) @@ -2208,7 +2212,7 @@ local db-value caching in the peer client (Phase 6.4). `transport.ss` bridges each `(std raft)` node's inbox/peer channels to TCP sockets so nodes can run in separate OS processes or on separate machines. -12/12 integration tests pass (`tests/test-transport.ss`, `make test-transport`). +23/23 integration tests pass (`tests/test-transport.ss`, `make test-transport`). ##### Architecture @@ -2227,7 +2231,7 @@ raft-node-A raft-node-B ↕ ↕ outbound fiber: outbound fiber: read proxy-ch-B → serialize read proxy-ch-A → serialize - → write to TCP conn to B → → write to TCP conn to A → + → write to persistent conn to B → → write to persistent conn to A → ↕ (TLS optional) ↕ (TLS optional) inbound fiber: inbound fiber: read TCP conn from B → deserialize read TCP conn from A → deserialize @@ -2241,14 +2245,14 @@ Only the plumbing beneath the channels changes. ``` Frame format (length-prefixed): - [4 bytes big-endian uint32: body length][body: FASL-encoded message] + [4 bytes big-endian uint32: body length][body: bounded safe-data envelope] -Messages (Raft vector messages, as-is from (std raft)): - #(request-vote term candidate-id last-log-index last-log-term) - #(vote-response term granted? voter-id) - #(append-entries term leader-id prev-idx prev-term entries commit-index) - #(append-response term success? follower-id match-index) - #(client-propose command reply-channel) ;; local only, never sent over wire +Messages (versioned pure-data forms produced and checked by `(std raft)`): + #(jerboa-raft-wire-v1 request-vote term candidate-id last-log-index last-log-term) + #(jerboa-raft-wire-v1 vote-response term granted? voter-id) + #(jerboa-raft-wire-v1 append-entries term leader-id prev-idx prev-term entries commit-index) + #(jerboa-raft-wire-v1 append-response term success? follower-id match-index) + ;; each entry is #(log-entry index term command) ``` `client-propose` is always local (client → leader via local channel). @@ -2262,7 +2266,7 @@ Only the four Raft RPC message types cross the wire. (import (jerboa prelude) (rename (only (chezscheme) make-time) (make-time chez-make-time)) (std net tcp) ;; tcp-listen, tcp-connect-binary, tcp-accept-binary - (std fasl) ;; fasl->bytevector, bytevector->fasl + (std safe-fasl) ;; bounded, non-executable data envelope (std misc channel) (std raft) ;; raft-node-*, raft-start!, raft-node-add-peer! (jerboa-db replication) @@ -2291,31 +2295,35 @@ Only the four Raft RPC message types cross the wire. ``` Key implementation details: -- **Wire protocol:** 4-byte big-endian uint32 length header + FASL body -- **Uni-directional connections:** node A dials B for A→B traffic; node B dials A for B→A traffic -- **Reconnect backoff:** 100ms → 5s (doubles on failure) to handle startup races +- **Wire protocol:** 4-byte big-endian uint32 length header + versioned safe-data body (1 MiB / 100,000-object limits) +- **Serialized persistent connections:** node A reuses one A→B connection; node B independently reuses one B→A connection +- **Reconnect backoff:** six bounded attempts with 100ms → 1.6s stop-aware delays to handle startup races - **`client-propose` dropped:** carries a live reply channel; never serialised over wire - **`raft-node-add-peer!`:** thread-safe; initialises `next-index`/`match-index` if node is already leader (prevents `send-heartbeats!` crash on new peer) -##### TLS (future) +##### TLS -The current TCP transport is plain text — suitable for localhost/loopback -clusters and trusted private networks. Production cross-datacenter deployments -should add mTLS via `(std net tls)`: +Plain TCP is available for localhost/loopback clusters. TLS is implemented via +`(std net tls)` and should be combined with authenticated transport identity for +production private-cluster deployments: ```scheme -;; Mutual TLS (mTLS) for inter-node auth: +;; TLS fixture; transport HMAC identity remains required off loopback: (def tls-cfg - (tls-config-with (default-tls-config) - :cert "/etc/jerboa-db/node.crt" - :key "/etc/jerboa-db/node.key" - :ca "/etc/jerboa-db/ca.crt")) + (make-tls-config + 'cert-file: "/etc/jerboa-db/node.crt" + 'key-file: "/etc/jerboa-db/node.key" + 'ca-file: "/etc/jerboa-db/ca.crt" + 'io-timeout-ms: 10000)) ;; Outbound: (tls-connect host port tls-cfg) instead of tcp-connect-binary -;; Inbound: (tls-accept tcp-server tls-cfg) instead of tcp-accept-binary +;; Inbound: bounded raw accept, admission, then tls-handshake-accepted ``` -For development and single-host clusters, plain TCP is fine. +For development and single-host clusters, plain TCP is restricted to loopback. +Any non-loopback Raft listener requires a 32-byte-or-longer transport +authentication key; TLS remains defense in depth for confidentiality and +certificate policy. #### 6.4 Peer Client (`peer.ss`) ✅ Functional (HTTP-based) @@ -2331,14 +2339,17 @@ operation is a network round-trip). ```scheme (import (jerboa-db peer)) +(def token (string-trim (read-file-string "/run/secrets/jerboa-db-token"))) + ;; Connect to a single server: -(def conn (connect-remote "http://localhost:8484")) +(def conn (connect-remote "http://localhost:8484" #f token)) ;; Connect to a cluster with automatic failover ;; (rotates through URLs on error with exponential backoff): (def conn (connect-remote* '("http://node-0:8484" "http://node-1:8484" - "http://node-2:8484"))) + "http://node-2:8484") + #f token)) ;; Transact — POSTs to /api/transact, returns tx-report alist: (remote-transact! conn @@ -2418,8 +2429,8 @@ These fixes are committed to `~/mine/jerboa` (commit c73d576). - ✅ Follower connections converge to leader state within 200ms (in-process) - ✅ `cluster-status` returns Raft fields + db basis-tx - ✅ `peer.ss` HTTP client — transact, query, pull with failover (multi-URL) -- ✅ TCP transport for multi-process/multi-host clusters (Phase 6.3) — 12/12 tests pass -- 🚧 TLS wrapping for production inter-node traffic +- ✅ TCP transport for multi-process/multi-host clusters (Phase 6.3) — 23/23 plain/auth tests pass +- ✅ Optional TLS wrapping for inter-node traffic - ✅ WebSocket tx-stream in peer client (Round 9, Phase 48) - ✅ Local db-value caching in peer client — basis-tx LRU cache (Round 9, Phase 49) @@ -2445,18 +2456,21 @@ These fixes are committed to `~/mine/jerboa` (commit c73d576). ```scheme (backup! conn "path/to/backup") -;; Serializes all indices + connection state as FASL + gzip +;; Creates a new mode-0600 file; existing paths/symlinks are rejected. +;; Serializes all indices + connection state in a bounded, versioned, +;; non-executable safe-data envelope, with optional gzip. ;; Point-in-time: backup is consistent at the latest committed tx -(restore! "path/to/backup" "path/to/new-db") -;; Copies data files, verifies integrity, opens new connection +(def restored (restore! "path/to/backup")) +;; Bounds compressed/expanded bytes, validates payloads, returns a new connection. ``` #### 7.3 CLI ```bash # Start server -jerboa-db serve --port 8484 --data /var/lib/jerboa-db +jerboa-db serve --port 8484 --data /var/lib/jerboa-db \ + --token-file /run/secrets/jerboa-db-token # Interactive query REPL jerboa-db repl --connect http://localhost:8484 @@ -2699,13 +2713,13 @@ Files: `lib/jerboa-db/replication.ss`, `lib/jerboa-db/cluster.ss`, | Read consistency levels | ✅ Done | `read-committed`, `read-latest`, `as-of-tx` | | Cluster status / introspection | ✅ Done | `cluster-status / cluster-leader?` | | In-process cluster test suite | ✅ Done | 6/6 tests passing (`tests/test-cluster.ss`) | -| TCP transport test suite | ✅ Done | 12/12 tests passing (`tests/test-transport.ss`) — startup, election, transact, replicate, status, stop | -| TCP transport (plain) | ✅ Done | `transport.ss` — 12/12 tests pass; length-framed FASL, reconnect backoff 100ms→5s, `raft-node-add-peer!` thread-safe | -| TLS wrapping | 🚧 TODO | Replace `tcp-connect-binary`/`tcp-accept-binary` with `(std net tls)` equivalents | +| TCP transport test suite | ✅ Done | 23/23 tests passing (`tests/test-transport.ss`) — schema rejection, startup, election, transact, replicate, authentication, admission/resource limits, duplicate rejection, idempotent stop | +| TCP transport (plain) | ✅ Done | `transport.ss` — bounded versioned safe-data frames, reusable peer connections, six-attempt stop-aware reconnect, `raft-node-add-peer!` thread-safe | +| TLS wrapping | ✅ Done | 7/7 TLS tests; reusable connection, multi-record transaction, bounded admission/lifetime, worker-zero shutdown | | Remote peer client (HTTP) | ✅ Done | `peer.ss` — connect-remote, remote-transact!, remote-q, remote-pull, multi-URL failover | | Remote peer — query-result cache | ✅ Done (Round 9 / Phase 49) | Basis-tx-keyed LRU; tunable capacity; `remote-cache-stats`, `remote-cache-clear!`, `remote-cache-set-capacity!` | | Remote peer — WebSocket stream | ✅ Stub (Round 9 / Phase 48) | `remote-tx-stream` connects to `/api/tx-stream` and dispatches frames to a callback | -| Remote peer — named DB routing | ✅ Done (Round 9 / Phase 47) | `(connect-remote url db-name)` routes through `/api/db/:name/...` | +| Remote peer — named DB routing | ✅ Done (Round 9 / Phase 47) | `(connect-remote url db-name token)` authenticates and routes through `/api/db/:name/...` | | Remote peer — `remote-entity` | ✅ Done (Round 9 / Phase 46) | `GET /api/entity/:eid`; alist round-trips correctly via dotted-pair (de)normalization | ### Polish (Phase 7) — ✅ MOSTLY COMPLETE --- a/lib/jerboa-db/backup.ss +++ b/lib/jerboa-db/backup.ss @@ -2,7 +2,7 @@ ;;; (jerboa-db backup) — Backup & Restore ;;; ;;; Snapshots the entire database (all four indices + schema + metadata) -;;; to a file using FASL encoding with optional gzip compression. +;;; to a file using the bounded DB storage codec with optional gzip compression. ;;; Restoration creates a fresh in-memory connection and replays all datoms. (library (jerboa-db backup) @@ -19,7 +19,11 @@ make-date make-time atom? meta) (jerboa prelude) - (only (std security taint) check-untainted!) + (only (std security taint) check-untainted! safe-delete-file) + (only (std native-loader) native-loader-ensure-libc-symbol!) + (std pkg gzipio) + (only (jerboa-db storage-codec) + db-storage-encode-bytevector db-storage-decode-bytevector) (jerboa-db datom) (jerboa-db schema) (jerboa-db index protocol) @@ -29,8 +33,18 @@ ;; ---- Magic header ---- ;; First 8 bytes of every backup file. - (def +backup-magic+ #vu8(74 68 66 75 49 48 48 48)) ;; "JDBU1000" + ;; Version 2 intentionally rejects legacy native-FASL backup payloads. + (def +backup-magic+ #vu8(74 68 66 75 50 48 48 48)) ;; "JDBU2000" (def +backup-max-datoms+ 10000000) + (def +backup-max-compressed-bytes+ (* 64 1024 1024)) + (def +backup-max-expanded-bytes+ (* 256 1024 1024)) + (def +backup-read-chunk-bytes+ (* 64 1024)) + (def +backup-max-storage-objects+ 4000000) + + (def _libc-loaded + (native-loader-ensure-libc-symbol! 'jerboa-db/backup "fchmod")) + (def c-fchmod + (foreign-procedure __collect_safe "fchmod" (int unsigned-32) int)) ;; ---- Lazy zlib loader ---- ;; Mirrors the LevelDB lazy loader pattern in core.sls. @@ -49,7 +63,7 @@ (set! zlib-loaded? #t)))) ;; ---- Schema serialization ---- - ;; Convert schema registry to a plain list for FASL portability. + ;; Convert schema registry to closed plain data for storage portability. (def (schema->plist schema) ;; Returns a list of (ident id vtype card unique index? comp? doc no-hist?) @@ -89,33 +103,95 @@ reg)) (def (valid-backup-datom? entry) - (and (list? entry) - (= (length entry) 5) + (and (equal? (bounded-proper-list-length entry 6) 5) (integer? (list-ref entry 0)) (integer? (list-ref entry 1)) (integer? (list-ref entry 3)) (boolean? (list-ref entry 4)))) + (def (valid-backup-schema-entry? entry) + (and (equal? (bounded-proper-list-length entry 10) 9) + (symbol? (list-ref entry 0)) + (integer? (list-ref entry 1)) + (>= (list-ref entry 1) 0) + (symbol? (list-ref entry 2)) + (symbol? (list-ref entry 3)) + (let ([unique (list-ref entry 4)]) + (or (not unique) (symbol? unique))) + (boolean? (list-ref entry 5)) + (boolean? (list-ref entry 6)) + (let ([doc (list-ref entry 7)]) + (or (not doc) (string? doc))) + (boolean? (list-ref entry 8)))) + + (def (bounded-proper-list-length value maximum) + ;; Returns the length or #f. Unlike an unbounded `length`, this also + ;; rejects cyclic/improper lists after a fixed amount of work. + (let loop ([rest value] [count 0]) + (cond + [(null? rest) count] + [(or (not (pair? rest)) (>= count maximum)) #f] + [else (loop (cdr rest) (+ count 1))]))) + (def (checked-backup-payload payload) - (unless (and (vector? payload) - (= (vector-length payload) 3) - (integer? (vector-ref payload 0)) - (list? (vector-ref payload 1)) - (<= (length (vector-ref payload 1)) +backup-max-datoms+) - (list? (vector-ref payload 2))) + (unless (and (vector? payload) (= (vector-length payload) 3)) (error 'restore! "invalid jerboa-db backup payload")) + (let ([datom-count + (bounded-proper-list-length + (vector-ref payload 1) (+ +backup-max-datoms+ 1))] + [schema-count + (bounded-proper-list-length (vector-ref payload 2) 100001)]) + (unless (and (integer? (vector-ref payload 0)) + datom-count (<= datom-count +backup-max-datoms+) + schema-count (<= schema-count 100000)) + (error 'restore! "invalid jerboa-db backup payload"))) (for-each (lambda (entry) (unless (valid-backup-datom? entry) (error 'restore! "invalid datom entry in backup"))) (vector-ref payload 1)) + (for-each + (lambda (entry) + (unless (valid-backup-schema-entry? entry) + (error 'restore! "invalid schema entry in backup"))) + (vector-ref payload 2)) payload) + (def (read-bounded-bytevector in maximum who) + (let-values ([(out get-result) (open-bytevector-output-port)]) + (let loop ([total 0]) + (let ([chunk (get-bytevector-n in + (min +backup-read-chunk-bytes+ + (+ 1 (- maximum total))))]) + (cond + [(eof-object? chunk) (get-result)] + [else + (let ([next (+ total (bytevector-length chunk))]) + (when (> next maximum) + (error who "backup input exceeds byte limit" maximum)) + (put-bytevector out chunk) + (loop next))]))))) + + (def (open-private-new-output path) + ;; Empty destinations may briefly inherit the umask, but no backup bytes + ;; are written until fchmod has made the held descriptor owner-only. + ;; Default file-options are create-new/fail-if-present, so symlinks and + ;; concurrent creators are rejected rather than followed or truncated. + (let ([out (open-file-output-port path + (file-options) + (buffer-mode block))]) + (let ([fd (port-file-descriptor out)]) + (unless (and fd (= (c-fchmod fd #o600) 0)) + (close-port out) + (guard (exn [#t (void)]) (safe-delete-file path)) + (error 'backup! "unable to set private backup permissions" path))) + out)) + ;; ---- backup! ---- ;; Serialize connection snapshot to output-path. ;; Format: - ;; [8 bytes magic] [1 byte: 0=raw, 1=gzip] [FASL payload] - ;; FASL payload is a vector: #(basis-tx next-eid schema-plist datoms-list) + ;; [8 bytes magic] [1 byte: 0=raw, 1=gzip] [safe storage envelope] + ;; The payload is a vector: #(basis-tx datoms-list schema-plist) ;; where datoms-list is a list of (e a v tx added?) 5-tuples. (def (backup! conn output-path) @@ -130,7 +206,7 @@ ;; We reconstruct it by reading all datoms and taking max eid + 1 [eavt (index-set-eavt indices)] [all-datoms (dbi-datoms eavt)] - ;; Serialize datoms as plain lists for FASL + ;; Serialize datoms as closed plain lists. [datom-list (map datom->list all-datoms)] [schema-pl (schema->plist schema)] ;; Compute next-eid from connection internals via db-stats @@ -139,20 +215,30 @@ ;; Store eavt-count as a marker; restore computes next-eid datom-list schema-pl)]) - ;; FASL-serialize payload to bytevector - (let-values ([(port get-bv) (open-bytevector-output-port)]) - (fasl-write payload port) - (let* ([raw-bv (get-bv)] - [use-gzip (and zlib-gzip #t)] - [data-bv (if use-gzip (zlib-gzip raw-bv) raw-bv)]) + (let* ([raw-bv + (db-storage-encode-bytevector + payload +backup-max-expanded-bytes+ + +backup-max-storage-objects+)] + [use-gzip (and zlib-gzip #t)] + [data-bv (if use-gzip (zlib-gzip raw-bv) raw-bv)]) + (when (> (bytevector-length data-bv) +backup-max-compressed-bytes+) + (error 'backup! "serialized backup exceeds storage byte limit" + +backup-max-compressed-bytes+)) ;; Write binary data: magic + compression flag + payload - (let ([out (open-file-output-port output-path - (file-options no-fail) - (buffer-mode block))]) - (put-bytevector out +backup-magic+) - (put-u8 out (if use-gzip 1 0)) - (put-bytevector out data-bv) - (close-port out)))))) + (let ([out (open-private-new-output output-path)]