triage-store: compute_triaged_ids bridge for triage-aware mode
Jaime Fournier
098ac998ad0fc69796bd451eebdc2282330df87b
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check analytics-check detect-check storage-check threats-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -50,6 +50,12 @@ triage-check: rust cd $(BUILD) && cargo build --release $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss +# Triage-aware mode: compute_triaged_ids over a live store, then prove detection +# (via exclude_event_ids) sees only the non-triaged events. Needs the native lib. +triage-store-check: rust + cd $(BUILD) && cargo build --release + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_store_check.ss + # Untyped per-host risk ranking + incident grouping, driving the host-risk-score # kernel. Checked against secmon's analytics vectors. analytics-check: rust @@ -83,6 +89,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/threats_check.ss + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_store_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `analytics` grouping + `group_incidents` | `jsecmon/analytics.ss` | ✅ **untyped layer** — per-host accumulation/sort/top-N driving the risk-score kernel, plus incident dedup/collapse; secmon analytics vectors pass (`make analytics-check`) | | `storage::detect_lolbin_cmdline` + `detect_dga_domain` | `jsecmon/detect.ss` | ✅ **untyped layer** — the kernel-driven detection rules: score every process_start cmdline (lolbin) / dns_query (dga) into anomalies above threshold. `make detect-check` runs the full events→detect→analytics pipeline; all three scoring kernels fire. Per-pattern label lists + label-level DGA dedup pending (need kernels that return the match breakdown). | | `triage` classifiers | `typed/triage.ss` | ✅ pure predicates (transient-unit?, phantom-rootkit-race?); vectors pass | -| `triage` engine (rules + dispatch) | `jsecmon/triage.ss` | ✅ **untyped layer** — all 18 false-positive rules + first-match engine, in secmon's exact RULES order, dispatch in ordinary Jerboa delegating byte/string classification to the typed kernels; 40 triage vectors pass (`make triage-check`), incl. the security-relevant negatives (non-sshd reading host keys, systemd impersonated from /tmp, unknown daemon reading passwd). Remaining: `compute_triaged_ids` (triage↔storage exclude_event_ids integration). | +| `triage` engine (rules + dispatch) | `jsecmon/triage.ss` | ✅ **untyped layer** — all 18 false-positive rules + first-match engine, in secmon's exact RULES order, dispatch in ordinary Jerboa delegating byte/string classification to the typed kernels; 40 triage vectors pass (`make triage-check`), incl. the security-relevant negatives (non-sshd reading host keys, systemd impersonated from /tmp, unknown daemon reading passwd). | +| `triage::compute_triaged_ids` (triage-aware mode) | `jsecmon/triage-store.ss` | ✅ **untyped layer** — the bridge above storage+triage: query every in-scope event, triage each, return the sorted benign/expected ID set to drop into a filter's `exclude_event_ids`. `make triage-store-check` proves the round-trip — detection then sees only the real attacks. | | `sigma` | — | ⏳ YAML import — I/O, untyped layer | | `psk::constant_time_eq` | `typed/psk.ss` | ✅ ported, vectors pass | | `psk::from_hex` (hex codec) | `typed/psk.ss` | ✅ hex encode + decode + 32-byte precondition; vectors pass (decode∘encode identity over all 256 byte values) | new file mode 100644 --- /dev/null +++ b/examples/triage_store_check.ss @@ -0,0 +1,53 @@ +;;; Round-trip check for triage-aware mode: store a mix of benign + real events, +;;; compute the triaged ID set, and confirm detection (via exclude_event_ids) +;;; then sees only the real ones. This is secmon's compute_triaged_ids contract. +;;; +;;; Needs the native sqlite lib on the loader path and the repo on --libdirs: +;;; (cd build/rust && cargo build --release) +;;; DYLD_LIBRARY_PATH=$JERBOA/lib scheme --libdirs "$JERBOA/lib:." \ +;;; --script examples/triage_store_check.ss + +(import (jerboa prelude) + (jsecmon storage) + (only (jsecmon triage) verdict?) + (jsecmon triage-store)) + +(def db (store-open ":memory:")) +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +;; store-event: db seq host source ts type severity pid pname summary data(JSON) +;; ids are AUTOINCREMENT in insert order: 1..5. +(store-event db 1 "h1" "s" 1000 "privilege_change" "critical" 10 "sshd" "" "{\"new_id\":0,\"id_type\":\"uid\"}") ;; 1 benign (sshd-privsep) +(store-event db 2 "h1" "s" 1001 "privilege_change" "critical" 11 "evil_binary" "" "{\"new_id\":0,\"id_type\":\"uid\"}") ;; 2 REAL privesc +(store-event db 3 "h1" "s" 1002 "mount_event" "high" 12 "(mandb)" "" "{\"source\":\"/dev/shm\"}") ;; 3 benign (private-mount) +(store-event db 4 "h1" "s" 1003 "reverse_shell" "critical" 13 "bash" "" "{\"remote_port\":4444}") ;; 4 REAL reverse shell +(store-event db 5 "h1" "s" 1004 "sensitive_file_access" "high" 14 "id" "" "{\"path\":\"/etc/passwd\"}") ;; 5 benign (lookup-tool) + +(displayln "triage-aware exclude-set (secmon compute_triaged_ids contract):") + +;; benign/expected events get collected; the two real attacks do not. +(check "triaged ids (all)" (compute-triaged-ids db (make-filter)) '(1 3 5)) +(check "triaged ids (host h1)" (compute-triaged-ids db (make-filter "host" "h1")) '(1 3 5)) + +;; per-row triage agrees with the engine. +(check "evil privesc is hot" (triage-row (car (query-events db (make-filter "pid" 11)))) #f) +(check "sshd privsep benign" (verdict? (triage-row (car (query-events db (make-filter "pid" 10))))) #t) + +;; the payoff: feed the triaged set back as exclude_event_ids and detection +;; only sees the real events (ids 2 and 4). +(let* ((triaged (compute-triaged-ids db (make-filter))) + (real (query-events db (make-filter "exclude_event_ids" triaged)))) + (check "non-triaged count" (length real) 2) + (check "non-triaged ids" + (list-sort < (map (lambda (r) (hash-get r "id")) real)) '(2 4))) + +(store-close db) +(newline) +(if (= fails 0) + (displayln "OK: compute-triaged-ids drives the triage-aware exclude set.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/triage-store.ss @@ -0,0 +1,63 @@ +#!chezscheme +;;; jsecmon triage-store — the bridge that makes triage actually cut noise. +;;; +;;; secmon's triage_row + compute_triaged_ids live in src/triage.rs because there +;;; the triage rules take a storage EventRow directly. We keep the two layers +;;; decoupled instead: (jsecmon storage) is pure SQLite, (jsecmon triage) is a +;;; pure classifier over an in-memory event struct, and THIS module — sitting +;;; above both — converts a queried row hash into a triage event and computes +;;; the benign/expected ID set. +;;; +;;; That set is exactly what `exclude_event_ids` wants: feed it back into a +;;; storage filter and detection/anomaly rules skip the triaged noise. This is +;;; secmon's triage-aware mode, the default for analyze detect/anomalies/watch. +;;; Verified against the round-trip in examples/triage_store_check.ss. + +(library (jsecmon triage-store) + (export row->triage-event triage-row compute-triaged-ids) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (only (jsecmon storage) query-events) + (only (jsecmon triage) make-event triage-event)) + + ;; A queried row hash (string keys; `data` already parsed to a sub-hash) -> + ;; the event struct triage classifies. process_name may be #f (SQL NULL); + ;; triage's pname helper turns that into "". + (def (row->triage-event row) + (make-event (or (hash-get row "event_type") "") + (hash-get row "process_name") + (hash-get row "data"))) + + ;; Triage a single queried row -> verdict | #f. + (def (triage-row row) + (triage-event (row->triage-event row))) + + ;; Re-scope a base filter for the triage sweep: triage EVERY in-scope event, + ;; not just the most recent page, and never recursively exclude the IDs we are + ;; about to compute (mirrors secmon clearing exclude_event_ids + limit=1e6). + (def (scan-filter base) + (let ((q (make-hash-table))) + (when base + (for-each + (lambda (k) + (unless (member k '("exclude_event_ids" "limit" "offset")) + (hash-put! q k (hash-get base k)))) + (hash-keys base))) + (hash-put! q "limit" 1000000) + q)) + + ;; Query all events matching base-filter, triage each, and return the sorted + ;; list of IDs triage classified as benign or expected — ready to drop into a + ;; filter's "exclude_event_ids". + (def (compute-triaged-ids db base-filter) + (let* ((rows (query-events db (scan-filter base-filter))) + (ids (filter-map (lambda (r) (and (triage-row r) (hash-get r "id"))) rows))) + (list-sort < ids))))