Use vendored jsqlite
ober
76c430a442d742ff6090fccb0ffe6238c82c8704
--- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Generated Rust crate (regenerate with `make rust`). /build/ +/vendor/ # Compiled binaries (rebuild with `make binaries`). /jsecmon-keygen --- a/Makefile +++ b/Makefile @@ -3,22 +3,47 @@ # Security-critical kernels live in typed/*.ss as Typed Jerboa and are compiled # to a Rust crate by the jerboa typed→rust backend, then verified against # secmon's own test vectors (tests/*.rs). -JERBOA ?= $(HOME)/mine/jerboa +VENDOR ?= $(CURDIR)/vendor +JERBOA_URL ?= https://git.sr.ht/~lisp/jerboa +JSQLITE_URL ?= https://git.sr.ht/~lisp/jsqlite +JERBOA ?= $(VENDOR)/jerboa +JSQLITE_REPO ?= $(VENDOR)/jsqlite +JSQLITE_SRC ?= $(JSQLITE_REPO)/src +JERBOA_NATIVE_FEATURES ?= tls,crypto SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check dtrace-runtime-check stealth-check ebpf-events-check ebpf-runtime-check proc-linux-check freebsd-parse-check event-meta-check config-check privdrop-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-auth-check monitor-kernel-check monitor-cron-check monitor-container-check monitor-rootkit-check monitor-podman-check monitor-selinux-check monitor-lateral-check monitor-webshell-check monitor-revshell-check monitor-persistence-check monitor-logtamper-check monitor-dns-check monitor-manager-check event-json-check collector-check protocol-check event-codec-check local-store-check collector-pull-check agent-server-check checks native-runtime keygen analyze collector agent binaries clean +.PHONY: vendor-deps ensure-jerboa ensure-jsqlite rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check dtrace-runtime-check stealth-check ebpf-events-check ebpf-runtime-check proc-linux-check freebsd-parse-check event-meta-check config-check privdrop-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-auth-check monitor-kernel-check monitor-cron-check monitor-container-check monitor-rootkit-check monitor-podman-check monitor-selinux-check monitor-lateral-check monitor-webshell-check monitor-revshell-check monitor-persistence-check monitor-logtamper-check monitor-dns-check monitor-manager-check event-json-check collector-check protocol-check event-codec-check local-store-check collector-pull-check agent-server-check checks native-runtime keygen analyze collector agent binaries clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). -LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" -# (jsecmon storage) loads libjerboa_native (rusqlite) through the dynamic loader, -# not by absolute path — point the loader at jerboa's lib dir (macOS + Linux). +LIBDIRS := "$(CURDIR):$(JSQLITE_SRC):$(JERBOA)/lib" +# Kept for non-SQL native modules; SQLite persistence is jsqlite and does not +# use libjerboa_native. LOADER_ENV := DYLD_LIBRARY_PATH="$(JERBOA)/lib:$$DYLD_LIBRARY_PATH" LD_LIBRARY_PATH="$(JERBOA)/lib:$$LD_LIBRARY_PATH" +vendor-deps: ensure-jerboa ensure-jsqlite + +ensure-jerboa: + @if [ ! -d "$(JERBOA)/.git" ]; then \ + mkdir -p "$(VENDOR)"; \ + git clone --depth 1 "$(JERBOA_URL)" "$(JERBOA)"; \ + fi + @if [ ! -x "$(SCHEME)" ]; then \ + $(MAKE) -C "$(JERBOA)" chez build; \ + fi + @test -x "$(SCHEME)" + +ensure-jsqlite: + @if [ ! -f "$(JSQLITE_SRC)/jsqlite/api.ss" ]; then \ + mkdir -p "$(VENDOR)"; \ + git clone --depth 1 "$(JSQLITE_URL)" "$(JSQLITE_REPO)"; \ + fi + @test -f "$(JSQLITE_SRC)/jsqlite/api.ss" + # Generate the Rust crate from the Typed Jerboa kernels, then drop in the # hand-written verification tests (the generator only writes src/ + Cargo.toml). -rust: +rust: ensure-jerboa cd $(JERBOA) && $(SCHEME) --libdirs lib --script support/typed-rust.ss \ $(CURDIR)/$(BUILD) $(addprefix $(CURDIR)/,$(TYPED)) @mkdir -p $(BUILD)/tests @@ -52,7 +77,7 @@ triage-check: rust # Triage-aware mode: compute_triaged_ids over a live store, then prove detection # (via exclude_event_ids) sees only the non-triaged events. Needs the native lib. -triage-store-check: rust +triage-store-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_store_check.ss @@ -68,23 +93,23 @@ detect-check: rust cd $(BUILD) && cargo build --release $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss -# Full persistence round-trip: store events to SQLite (via std db sqlite-native), +# Full persistence round-trip: store events to SQLite via jsqlite, # query them back as row hashes, then run the same detect -> analytics pipeline # over the queried rows. Proves storage composes with the rest of the layer. -storage-check: rust +storage-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss # storage::entity_timeline: the per-category WHERE-group builder (process/ip/ # user/domain field routing) plus the assembled query over a live store. Pure -# storage layer, so no rust dep -- only the native sqlite loader. -entity-check: +# storage layer, so no rust dep and no native SQLite loader. +entity-check: vendor-deps $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/entity_check.ss # The SQL-aggregation threat detectors (brute_force, credential_stuffing, # dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) over a live store, # checked against secmon's run_detections test vectors. Needs the native lib too. -threats-check: rust +threats-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/threats_check.ss @@ -92,7 +117,7 @@ threats-check: rust # impossible_travel detector over a live store. SECMON_GEOIP_CSV points the rule # at the fixture, exactly as secmon reads its own CSV from an env-configured path. GEOIP_CSV := $(CURDIR)/examples/fixtures/geoip_sample.csv -geoip-check: rust +geoip-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) SECMON_GEOIP_CSV="$(GEOIP_CSV)" $(SCHEME) --libdirs $(LIBDIRS) --script examples/geoip_check.ss @@ -104,7 +129,7 @@ sigma-check: # User-supplied YAML detection rules (threshold/distinct/sequence/match) run # over a live store, checked against secmon's yaml_rules behaviour. Native lib. -yaml-rules-check: +yaml-rules-check: vendor-deps $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/yaml_rules_check.ss # The agent's encrypted event ring buffer (secmon src/buffer/ring.rs): FIFO + @@ -316,7 +341,7 @@ event-codec-check: # Local encrypted event store (secmon src/local_store.rs): SQLite schema, # load-or-generate 32-byte key, metadata columns, AES-GCM encrypted # SecurityEvent payloads, ordered polling, and cleanup semantics. -local-store-check: rust +local-store-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/local_store_check.ss @@ -329,7 +354,7 @@ collector-pull-check: rust # Agent-side poll server: PSK challenge verification, request dispatch, ECIES # buffer storage, and a real loopback collector-style socket exchange. -agent-server-check: rust +agent-server-check: rust ensure-jsqlite cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/agent_server_check.ss @@ -504,12 +529,12 @@ event-json-check: # Agent collector loop (secmon agent storage path): boot + a tick driven through # the monitors into a real temp SQLite store, then queried back to assert events # are serialized, summarized, pid/name-extracted, and persisted. No typed kernels -# (no rust dep) -- only the native sqlite loader, like entity-check. -collector-check: +# (no rust dep) -- only the jsqlite storage layer, like entity-check. +collector-check: vendor-deps $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_check.ss # Everything that runs through the Jerboa side of the bridge, one shot. -checks: kernels-check +checks: kernels-check ensure-jsqlite $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss @@ -592,35 +617,33 @@ checks: kernels-check # entry point, bundles the Chez kernel + stdlib boot image, and STATICALLY links # the Rust kernels from libjerboa_typed_generated.a (force_load + export_dynamic, # so the jt_* C-ABI symbols resolve in-process via dlsym). The result needs no -# scheme, no .ss, and no jsecmon kernel .dylib at runtime. SQLite-using binaries -# preload Jerboa's native dylib by absolute path and register its SQLite FFI -# symbols, so they do not need DYLD_LIBRARY_PATH/LD_LIBRARY_PATH. -native-runtime: - cd $(JERBOA)/jerboa-native-rs && cargo build --release --features full - $(MAKE) -C $(JERBOA) native - -keygen: rust +# scheme, no .ss, and no jsecmon kernel .dylib at runtime. SQLite persistence +# uses vendored jsqlite, so no native SQLite feature is built or registered. +native-runtime: ensure-jerboa + cd $(JERBOA)/jerboa-native-rs && cargo build --release --no-default-features --features "$(JERBOA_NATIVE_FEATURES)" + +keygen: rust ensure-jsqlite cd $(BUILD) && cargo build --release JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/keygen.ss jsecmon-keygen # secmon-analyze: query/detect/triage/risk over the SQLite store. Statically # links the Rust kernels (lolbin/dga scoring, calendar) like keygen; SQLite is -# provided by the preloaded Jerboa native runtime. -analyze: rust native-runtime +# provided by vendored jsqlite. +analyze: rust native-runtime ensure-jsqlite cd $(BUILD) && cargo build --release JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/analyze.ss jsecmon-analyze # secmon-collector: pull/status/watch over the PSK-encrypted protocol; decrypts # ECIES events and prints (human/NDJSON) and/or stores them. Statically links the # Rust crypto kernels (ECIES/PSK/HKDF/AES-GCM); libc sockets resolve in-process. -# SQLite is available through the same preloaded native runtime when --db is used. -collector: rust native-runtime +# SQLite is available through vendored jsqlite when --db is used. +collector: rust native-runtime ensure-jsqlite cd $(BUILD) && cargo build --release JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/collector.ss jsecmon-collector # secmon-agent: Linux polling monitors + encrypted event buffer + poll server. # It loads the collector public key and PSK at runtime, never embeds secrets. -agent: rust native-runtime +agent: rust native-runtime ensure-jsqlite cd $(BUILD) && cargo build --release JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/agent.ss jsecmon-agent --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # jsecmon -A pure-[Jerboa](../jerboa) reimplementation of [`secmon`](../secmon), a host +A pure-Jerboa reimplementation of secmon, a host security monitor. The goal: no hand-written Rust or C. Security-critical kernels (crypto, secrets, detection scoring) are written in **Typed Jerboa** and compiled to Rust by jerboa's typed→rust backend; the rest is ordinary Jerboa. @@ -103,7 +103,7 @@ root must be on the libdir path so the import resolves to `jsecmon/kernels.ss`. `examples/kernels_check.ss` re-checks the secmon parity vectors through these wrappers (`make kernels-check`). -`make` needs a built jerboa checkout at `$JERBOA` (default `~/mine/jerboa`), +`make` needs a built Jerboa checkout at `$JERBOA` (default `vendor/jerboa`), whose `.chez/bin/scheme` and `support/typed-rust.ss` drive the backend. ## Binaries @@ -166,7 +166,7 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `crypto` primitives (SHA256/HMAC/HKDF/x25519/AES-256-GCM) | `typed/crypto.ss` | ✅ **typed kernel** — FFI-delegated to vetted RustCrypto crates (sha2/hmac/hkdf/x25519-dalek/aes-gcm), never reimplemented; the backend emits the `use … ;` block per primitive and content-scans the Cargo deps. Vector-verified against NIST SP 800-38D GCM, RFC 7748/5869/4231, FIPS 180-4 (`make test`). | | `psk` key-derivation + proof + transport (`from_bytes`, `compute_proof`, `encrypt_transport`) | `typed/psk.ss` + `jsecmon/crypto-psk.ss` | ✅ **typed kernel + untyped orchestration** — HKDF-derived auth/transport keys, the SHA256 challenge proof + constant-time `verify_proof`, and AES-256-GCM transport (12-byte nonce prepended) are typed kernels (`make test`); `(jsecmon crypto-psk)` adds the effects they omit — `transport-encrypt` draws a random nonce, `generate-challenge`/`respond-to-challenge`/`verify-response` add the random challenge nonce + clock + freshness check (`now` injected for testability). `make crypto-psk-check` pins the FFI path to the `psk_vectors.rs` digests then exercises round-trip / randomization / tamper→#f / wrong-key→#f / stale→#f. | | `crypto::ecies` (x25519 ECDH + HKDF + AES-GCM) | `typed/ecies.ss` + `jsecmon/crypto-ecies.ss` | ✅ **typed kernel + untyped orchestration** — deterministic `ecies-seal`/`-open` are typed kernels (parity vs an independent Python impl, `ecies_vectors.rs`); `(jsecmon crypto-ecies)` adds `ecies-generate-keypair` + `ecies-encrypt` (random ephemeral keypair + nonce) + `ecies-decrypt`. Frame = secmon's bincode `EncryptedPayload` layout: `ephemeral_public(32) ‖ nonce(12) ‖ ciphertext_len(u64le) ‖ ciphertext`; decrypt also accepts the older jsecmon bare concat for local backward compatibility. `make crypto-ecies-check` pins x25519 to RFC 7748 + ecies-seal to the reference vector, then bincode-frame layout / round-trip / randomization / wrong-recipient→#f / tamper→#f. | -| `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). `entity-where`/`entity-timeline` port `storage::entity_timeline`: the per-category WHERE-group builder (Process = `process_name LIKE` + six json fields with LIKE `%value%`; Ip/User/Domain = exact `json_extract` `=` over their field sets) plus the assembled oldest-first query, with the filter clauses continuing the positional `?N` numbering after the category binds (`build-where` gained an optional start index). `make entity-check` pins each category's exact SQL fragment + binds and routes/orders/narrows/limits over a live store (18 cases). | +| `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on vendored `jsqlite`: secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). `entity-where`/`entity-timeline` port `storage::entity_timeline`: the per-category WHERE-group builder (Process = `process_name LIKE` + six json fields with LIKE `%value%`; Ip/User/Domain = exact `json_extract` `=` over their field sets) plus the assembled oldest-first query, with the filter clauses continuing the positional `?N` numbering after the category binds (`build-where` gained an optional start index). `make entity-check` pins each category's exact SQL fragment + binds and routes/orders/narrows/limits over a live store (18 cases). | | `storage` SQL-aggregation detectors (brute_force, credential_stuffing, dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `run_detections` family: the time-bucket GROUP BY/HAVING rules and the two 5-min sliding-window rules, run as SQL (json_extract) over a `(jsecmon storage)` handle. `make threats-check` reproduces secmon's six detection-rule test vectors; the companion sequence/statistical families are covered by the rows below. | | `storage` sequence/chain detectors (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_sequence_pair` family: event A then event B within a window on the same host (auth-success→priv-esc /5min, reverse-shell/webshell→persistence /1h, any-critical→log-tampering /1h, shell→lateral /1h). Reproduces secmon's chain test vectors incl. the outside-window negative. | | `storage` time-window aggregates (frequency_spike, severity_cluster, off_hours, kill_chain) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's full `detect_anomalies` family: per-(host,event_type) hour count 3x above its own average, 5+ crit/high on a host /5min, crit/high outside 08:00-18:00 UTC weekday (SQLite `strftime`), and 3+ distinct kill-chain phases /1h. `run-anomaly-detections` is the dispatcher (frequency_spike first, as secmon runs it). `make threats-check` covers each with threshold/negative cases. | --- a/bin/analyze.ss +++ b/bin/analyze.ss @@ -19,7 +19,7 @@ ;;; ;;; Entry point built by build-binary.ss → a standalone native binary; argv comes ;;; in via (command-line-arguments) = (command db remaining…), i.e. secmon's -;;; args[1..]. SQLite still resolves libjerboa_native at runtime via the loader. +;;; args[1..]. SQLite persistence resolves through vendored jsqlite. (import (except (chezscheme) make-hash-table hash-table? --- a/build-binary.ss +++ b/build-binary.ss @@ -14,10 +14,9 @@ ;;; them). kernels.ss does (load-shared-object #f) in this mode. ;;; ;;; Rust staticlibs cannot safely link both the typed kernels and Jerboa's native -;;; runtime archive: each archive brings its own Rust std/panic objects. For -;;; SQLite, the generated C main dlopens Jerboa's native dylib by absolute path -;;; and registers the SQLite entries with Chez, so the binary does not require -;;; DYLD_LIBRARY_PATH/LD_LIBRARY_PATH at startup. +;;; runtime archive: each archive brings its own Rust std/panic objects. SQLite +;;; persistence uses the vendored jsqlite library in Scheme, so this build path +;;; does not register native SQLite symbols. ;;; ;;; Modeled on jerboa-gitsafe/build-binary.ss. Whole-program optimization inlines ;;; every library that ships a .wpo; libraries that ship only a precompiled .so @@ -124,10 +123,15 @@ (define jerboa-dir (or (getenv "JERBOA_HOME") - (let ([sib (format "~a/../jerboa" (current-directory))]) (and (file-exists? sib) sib)) - (let ([m (format "~a/mine/jerboa" home)]) (and (file-exists? m) m)) + (let ([v (format "~a/vendor/jerboa" (current-directory))]) + (and (file-exists? v) v)) (begin (display "Error: Cannot find Jerboa. Set JERBOA_HOME.\n") (exit 1)))) +(define jsqlite-src + (let ([dir (or (getenv "JSQLITE_DIR") + (format "~a/vendor/jsqlite" (current-directory)))]) + (format "~a/src" dir))) + ;; Jerboa bundles its own Chez at $JERBOA/.chez/lib/csv*/<mt>; prefer it so the ;; binary is built against the exact kernel that compiled the libraries. (define chez-dir @@ -147,9 +151,8 @@ (exit 1)) (define needs-native? - (or (member entry '("bin/analyze.ss" "bin/collector.ss" "bin/agent.ss")) - (let ([v (getenv "JSECMON_REQUIRE_NATIVE")]) - (and v (not (string=? v "")) (not (string=? v "0")))))) + (let ([v (getenv "JSECMON_REQUIRE_NATIVE")]) + (and v (not (string=? v "")) (not (string=? v "0"))))) (define native-lib (and needs-native? @@ -163,36 +166,13 @@ [(file-exists? lib-dylib) lib-dylib] [else #f])))) (when (and needs-native? (not native-lib)) - (printf "Error: Jerboa native dylib not found under ~a\n (run: cd ~a/jerboa-native-rs && cargo build --release --features full)\n" + (printf "Error: Jerboa native dylib not found under ~a\n (run: cd ~a/jerboa-native-rs && cargo build --release --no-default-features --features tls,crypto)\n" jerboa-dir jerboa-dir) (exit 1)) -;; The binaries currently need SQLite from (std db sqlite-native). Registering -;; these names makes JERBOA_STATIC=1 work after the C main preloads Jerboa's -;; native dylib by absolute path. -(define static-native-symbols - '("jerboa_sqlite_open" - "jerboa_sqlite_close" - "jerboa_sqlite_exec" - "jerboa_sqlite_prepare" - "jerboa_sqlite_finalize" - "jerboa_sqlite_reset" - "jerboa_sqlite_bind_int" - "jerboa_sqlite_bind_double" - "jerboa_sqlite_bind_text" - "jerboa_sqlite_bind_blob" - "jerboa_sqlite_bind_null" - "jerboa_sqlite_step" - "jerboa_sqlite_column_count" - "jerboa_sqlite_column_type" - "jerboa_sqlite_column_int" - "jerboa_sqlite_column_double" - "jerboa_sqlite_column_text" - "jerboa_sqlite_column_blob" - "jerboa_sqlite_column_name" - "jerboa_sqlite_last_insert_rowid" - "jerboa_sqlite_changes" - "jerboa_sqlite_errmsg")) +;; SQLite persistence now uses jsqlite, so no sqlite-native symbols are +;; registered here. Keep the list available for opt-in non-SQL native modules. +(define static-native-symbols '()) ;; Some Jerboa stdlib modules also check JERBOA_STATIC. The collector imports ;; (std net tcp), so register the libc entries it uses when static mode is on. @@ -229,6 +209,7 @@ ;; libraries resolve from the repo root and the jerboa stdlib (library-directories (append (list (cons (current-directory) (current-directory)) + (cons jsqlite-src jsqlite-src) (cons (format "~a/lib" jerboa-dir) (format "~a/lib" jerboa-dir))) (library-directories))) --- a/examples/geoip_check.ss +++ b/examples/geoip_check.ss @@ -6,10 +6,8 @@ ;;; countries inside the gap fires; outside the gap, same country, and a private ;;; source IP do not. ;;; -;;; Needs the native sqlite lib on the loader path, the repo on --libdirs, and -;;; SECMON_GEOIP_CSV pointing at the fixture (the Makefile geoip-check sets all): -;;; DYLD_LIBRARY_PATH=$JERBOA/lib SECMON_GEOIP_CSV=examples/fixtures/geoip_sample.csv \ -;;; scheme --libdirs "$JERBOA/lib:." --script examples/geoip_check.ss +;;; Needs vendored jsqlite on --libdirs and SECMON_GEOIP_CSV pointing at the +;;; fixture (the Makefile geoip-check sets both). (import (jerboa prelude) (jsecmon storage) --- a/examples/local_store_check.ss +++ b/examples/local_store_check.ss @@ -9,8 +9,7 @@ (import (jerboa prelude) (jsecmon local-store) - (only (jsecmon event-codec) encode-security-event) - (std db sqlite-native)) + (only (jsecmon event-codec) encode-security-event)) (def fails 0) (def (check name got want) --- a/examples/triage_store_check.ss +++ b/examples/triage_store_check.ss @@ -2,10 +2,7 @@ ;;; compute the triaged ID set, and confirm detection (via exclude_event_ids) ;;; then sees only the real ones. This is secmon's compute_triaged_ids contract. ;;; -;;; Needs the native sqlite lib on the loader path and the repo on --libdirs: -;;; (cd build/rust && cargo build --release) -;;; DYLD_LIBRARY_PATH=$JERBOA/lib scheme --libdirs "$JERBOA/lib:." \ -;;; --script examples/triage_store_check.ss +;;; Needs vendored jsqlite on --libdirs and the typed kernels built. (import (jerboa prelude) (jsecmon storage) --- a/jsecmon/agent-server.ss +++ b/jsecmon/agent-server.ss @@ -26,7 +26,7 @@ iota 1+ 1- partition make-date make-time) - (except (jerboa prelude) meta atom?) + (except (jerboa prelude) meta atom? tcp-listen tcp-close) (only (std net tcp) tcp-listen tcp-accept-binary tcp-server-port tcp-close) (only (jsecmon frame) frame-encode frame-read-length) (only (jsecmon buffer) --- a/jsecmon/local-store.ss +++ b/jsecmon/local-store.ss @@ -27,34 +27,37 @@ iota 1+ 1- partition make-date make-time) - (except (jerboa prelude) meta atom?) + (except (jerboa prelude) meta atom? + sqlite-open sqlite-close sqlite-exec sqlite-execute + sqlite-query) (only (jsecmon event-codec) encode-security-event decode-security-event) (only (jsecmon event-meta) event-severity-u8) (only (jsecmon kernels) psk-transport-seal psk-transport-open) (std crypto random) - (std db sqlite-native) + (jsecmon sqlite) (only (std os shell) shell-quote)) (defstruct local-store (db key)) (defstruct local-stored-event (seq timestamp-ms severity event-type nonce encrypted)) (def schema-ddl - (string-append - "PRAGMA journal_mode = WAL;" - "PRAGMA synchronous = NORMAL;" - "PRAGMA busy_timeout = 5000;" - "CREATE TABLE IF NOT EXISTS local_events (" - " seq INTEGER PRIMARY KEY," - " timestamp_ms INTEGER NOT NULL," - " severity INTEGER NOT NULL," - " event_type TEXT NOT NULL," - " nonce BLOB NOT NULL," - " encrypted BLOB NOT NULL" - ");" - "CREATE INDEX IF NOT EXISTS idx_local_ts ON local_events(timestamp_ms);" - "CREATE INDEX IF NOT EXISTS idx_local_sev ON local_events(severity);" - "CREATE INDEX IF NOT EXISTS idx_local_type ON local_events(event_type);")) + (list + "PRAGMA journal_mode = WAL" + "PRAGMA synchronous = NORMAL" + "PRAGMA busy_timeout = 5000" + (string-append + "CREATE TABLE IF NOT EXISTS local_events (" + " seq INTEGER PRIMARY KEY," + " timestamp_ms INTEGER NOT NULL," + " severity INTEGER NOT NULL," + " event_type TEXT NOT NULL," + " nonce BLOB NOT NULL," + " encrypted BLOB NOT NULL" + ")") + "CREATE INDEX IF NOT EXISTS idx_local_ts ON local_events(timestamp_ms)" + "CREATE INDEX IF NOT EXISTS idx_local_sev ON local_events(severity)" + "CREATE INDEX IF NOT EXISTS idx_local_type ON local_events(event_type)")) (def (read-file-bytes path) (call-with-port (open-file-input-port path) @@ -99,7 +102,7 @@ (def (local-store-open db-path key-path) (let* ((key (load-or-generate-local-key key-path)) (db (sqlite-open db-path))) - (sqlite-exec db schema-ddl) + (for-each (lambda (sql) (sqlite-exec db sql)) schema-ddl) (make-local-store db key))) (def (local-store-close store) new file mode 100644 --- /dev/null +++ b/jsecmon/sqlite.ss @@ -0,0 +1,54 @@ +#!chezscheme +;;; jsecmon sqlite adapter over the Jerboa-native jsqlite engine. +;;; +;;; The older sqlite-native binding returned query rows as alists keyed by +;;; column name. jsqlite's public API returns positional lists, so this module +;;; preserves the storage layer's row contract while removing the FFI backend. + +(library (jsecmon sqlite) + (export sqlite-open sqlite-close sqlite-exec sqlite-execute sqlite-query + sqlite-changes) + (import (chezscheme) + (prefix (jsqlite api) japi:) + (prefix (jsqlite constants) jconst:) + (prefix (jsqlite value) jval:)) + + (define sqlite-open japi:sqlite-open) + (define sqlite-close japi:sqlite-close) + (define sqlite-changes japi:sqlite-changes) + + (define (->sql-value v) + (cond + [(eq? v #f) jval:sql-null] + [(eq? v #t) 1] + [else v])) + + (define (from-sql-value v) + (if (jval:sql-null? v) #f v)) + + (define (sqlite-exec db sql . vals) + (apply japi:sqlite-exec db sql (map ->sql-value vals))) + + (define sqlite-execute sqlite-exec) + + (define (bind-args! stmt vals) + (let loop ([i 1] [vs vals]) + (unless (null? vs) + (japi:sqlite-bind! stmt i (->sql-value (car vs))) + (loop (+ i 1) (cdr vs))))) + + (define (sqlite-query db sql . vals) + (let ([stmt (japi:sqlite-prepare db sql)]) + (bind-args! stmt vals) + (let ([names (japi:sqlite-columns stmt)]) + (let loop ([acc '()]) + (let ([rc (japi:sqlite-step stmt)]) + (cond + [(= rc jconst:SQLITE_ROW) + (loop (cons (map cons names (map from-sql-value (japi:sqlite-row stmt))) acc))] + [(= rc jconst:SQLITE_DONE) + (japi:sqlite-finalize stmt) + (reverse acc)] + [else + (japi:sqlite-finalize stmt) + (reverse acc)]))))))) --- a/jsecmon/storage.ss +++ b/jsecmon/storage.ss @@ -1,11 +1,10 @@ #!chezscheme ;;; jsecmon storage — the event persistence backbone, untyped orchestration. ;;; -;;; This is secmon's src/storage/mod.rs: a SQLite-backed event store. It is pure +;;; This is secmon's src/storage/mod.rs: a jsqlite-backed event store. It is pure ;;; I/O + SQL, so it stays in the untyped layer (no typed kernel touches a -;;; database). It runs on (std db sqlite-native) — the rusqlite-backed native -;;; binding that ships with jerboa — NOT (std db sqlite), whose chez backend is -;;; undefined in this checkout. +;;; database). It runs on the Jerboa-native (jsecmon sqlite) adapter over +;;; jsqlite, not the sqlite-native FFI backend. ;;; ;;; The contract that makes the whole port compose: query-events returns events ;;; as *row hash tables with string keys* — exactly the shape (jsecmon detect), @@ -35,41 +34,46 @@ iota 1+ 1- partition make-date make-time) - (except (jerboa prelude) meta atom?) + (except (jerboa prelude) meta atom? + sqlite-open sqlite-close sqlite-exec sqlite-execute + sqlite-query) (jsecmon ioc) - (std db sqlite-native)) + (jsecmon sqlite)) ;; ── schema (secmon's events table + core indexes + collector_state) ───────── (def schema-ddl - (string-append - "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;" - "CREATE TABLE IF NOT EXISTS events (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " seq INTEGER NOT NULL," - " host TEXT NOT NULL," - " source TEXT NOT NULL DEFAULT ''," - " timestamp_ms INTEGER NOT NULL," - " event_type TEXT NOT NULL," - " severity TEXT NOT NULL," - " pid INTEGER," - " process_name TEXT," - " summary TEXT NOT NULL DEFAULT ''," - " data TEXT NOT NULL," - " UNIQUE(host, source, seq));" - "CREATE INDEX IF NOT EXISTS idx_ts ON events(timestamp_ms);" - "CREATE INDEX IF NOT EXISTS idx_host_ts ON events(host, timestamp_ms);" - "CREATE INDEX IF NOT EXISTS idx_type_ts ON events(event_type, timestamp_ms);" - "CREATE INDEX IF NOT EXISTS idx_severity_ts ON events(severity, timestamp_ms);" - "CREATE INDEX IF NOT EXISTS idx_pid ON events(pid) WHERE pid IS NOT NULL;" - "CREATE TABLE IF NOT EXISTS collector_state (" - " source TEXT PRIMARY KEY," - " last_seq INTEGER NOT NULL DEFAULT 0," - " last_seen INTEGER NOT NULL," - " hostname TEXT);")) + (list + "PRAGMA journal_mode=WAL" + "PRAGMA synchronous=NORMAL" + (string-append + "CREATE TABLE IF NOT EXISTS events (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " seq INTEGER NOT NULL," + " host TEXT NOT NULL," + " source TEXT NOT NULL DEFAULT ''," + " timestamp_ms INTEGER NOT NULL," + " event_type TEXT NOT NULL," + " severity TEXT NOT NULL," + " pid INTEGER," + " process_name TEXT," + " summary TEXT NOT NULL DEFAULT ''," + " data TEXT NOT NULL," + " UNIQUE(host, source, seq))") + "CREATE INDEX IF NOT EXISTS idx_ts ON events(timestamp_ms)" + "CREATE INDEX IF NOT EXISTS idx_host_ts ON events(host, timestamp_ms)" + "CREATE INDEX IF NOT EXISTS idx_type_ts ON events(event_type, timestamp_ms)" + "CREATE INDEX IF NOT EXISTS idx_severity_ts ON events(severity, timestamp_ms)" + "CREATE INDEX IF NOT EXISTS idx_pid ON events(pid) WHERE pid IS NOT NULL" + (string-append + "CREATE TABLE IF NOT EXISTS collector_state (" + " source TEXT PRIMARY KEY," + " last_seq INTEGER NOT NULL DEFAULT 0," + " last_seen INTEGER NOT NULL," + " hostname TEXT)"))) (def (store-open path) (let ((db (sqlite-open path))) - (sqlite-exec db schema-ddl) + (for-each (lambda (sql) (sqlite-exec db sql)) schema-ddl) db)) (def (store-close db) (sqlite-close db)) --- a/jsecmon/threats.ss +++ b/jsecmon/threats.ss @@ -18,7 +18,7 @@ ;;; ;;; The bucket rules are GROUP BY ... HAVING; the two sliding-window rules pull ;;; ordered rows and walk them in Jerboa exactly as secmon walks its Vec. Uses -;;; json_extract (json1 is present in the rusqlite this binds). Anomalies are the +;;; json_extract through the Jerboa-native jsqlite engine. Anomalies are the ;;; same row-hash shape (jsecmon analytics) consumes. Verified in ;;; examples/threats_check.ss against secmon's own detection-rule test vectors. @@ -40,8 +40,10 @@ iota 1+ 1- partition make-date make-time) - (except (jerboa prelude) meta atom?) - (std db sqlite-native) + (except (jerboa prelude) meta atom? + sqlite-open sqlite-close sqlite-exec sqlite-execute + sqlite-query) + (jsecmon sqlite) (only (jsecmon storage) make-filter build-where) (only (jsecmon geoip) load-csv geo-lookup geo-country geo-asn)) --- a/jsecmon/yaml-rules.ss +++ b/jsecmon/yaml-rules.ss @@ -36,8 +36,10 @@ iota 1+ 1- partition make-date make-time) - (except (jerboa prelude) meta atom?) - (std db sqlite-native) + (except (jerboa prelude) meta atom? + sqlite-open sqlite-close sqlite-exec sqlite-execute + sqlite-query) + (jsecmon sqlite) (std text yaml) (only (jsecmon storage) make-filter build-where)) @@ -45,7 +47,7 @@ (def (a-of row k) (let ((e (assoc k row))) (and e (cdr e)))) (def (a-str row k) (let ((v (a-of row k))) (if (string? v) v ""))) (def (a-num row k) (let ((v (a-of row k))) (if (number? v) v 0))) - (def (val->str v) ;; rusqlite value_to_string analogue + (def (val->str v) ;; SQLite value-to-string analogue (cond ((eq? v #f) "<null>") ((number? v) (number->string v)) ((string? v) v) (else (format "~a" v))))