Add untyped (jsecmon analytics) risk ranking + incident grouping
Jaime Fournier <jaimef@linbsd.org>
76ad9e5854c424c901abc55eb6ec9edd0b04ad8a
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check clean +.PHONY: rust test ffi-demo kernels-check triage-check analytics-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -47,5 +47,16 @@ triage-check: rust cd $(BUILD) && cargo build --release $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss +# Untyped per-host risk ranking + incident grouping, driving the host-risk-score +# kernel. Checked against secmon's analytics vectors. +analytics-check: rust + cd $(BUILD) && cargo build --release + $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss + +# Everything that runs through the Jerboa side of the bridge, one shot. +checks: kernels-check + $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss + clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ make test # regenerate, then cargo test against secmon's vectors make ffi-demo # build the cdylib + drive two kernels from a Jerboa script make kernels-check # exercise the (jsecmon kernels) library against the vectors make triage-check # verify the untyped (jsecmon triage) engine vs secmon vectors +make analytics-check # verify untyped risk-ranking + incident grouping vs vectors +make checks # all three Jerboa-side checks in one shot ``` ### The C ABI bridge @@ -62,7 +64,8 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `dga::score_domain` | `typed/dga.ss` | ✅ full: lowercase + dot-trim + benign-suffix + label split + score; vectors pass (diagnostic `reasons` list pending) | | `&str` ops (lowercase/ends_with/starts_with/contains/split/whole-word) | `typed/strbytes.ss` | ✅ Bytes toolkit, vectors pass — shared by dga/lolbin/sigma | | `lolbin::score` + `severity` | `typed/lolbin.ss` | ✅ full 25-pattern table + severity buckets; vectors pass (JSON-cmdline parse stays in untyped wrapper; diagnostic match list with caller) | -| `analytics::compute_host_risks` | `typed/analytics.ss` | ✅ risk-score kernel (clamped weighted sum); vectors pass (host grouping/sort/top-N stays untyped) | +| `analytics::compute_host_risks` | `typed/analytics.ss` | ✅ risk-score kernel (clamped weighted sum); vectors pass | +| `analytics` grouping + `group_incidents` | `jsecmon/analytics.ss` | ✅ **untyped layer** — per-host accumulation/sort/top-N driving the risk-score kernel, plus incident dedup/collapse; secmon analytics vectors pass (`make analytics-check`) | | `triage` classifiers | `typed/triage.ss` | ✅ pure predicates (transient-unit?, phantom-rootkit-race?); vectors pass | | `triage` engine (rules + dispatch) | `jsecmon/triage.ss` | ✅ **untyped layer** — 6 false-positive rules + first-match engine, dispatch in ordinary Jerboa delegating byte/string classification to the typed kernels; 11 secmon triage vectors pass (`make triage-check`). Remaining rules are mechanical follow-on. | | `sigma` | — | ⏳ YAML import — I/O, untyped layer | new file mode 100644 --- /dev/null +++ b/examples/analytics_check.ss @@ -0,0 +1,68 @@ +;;; Parity check for the untyped (jsecmon analytics) layer. +;;; +;;; Reproduces secmon's analytics tests (src/analytics.rs #[test] mod): +;;; risk_prefers_chains_over_noise, incidents_collapse_by_key — plus an exact +;;; host-b score assertion (34) confirming the typed host-risk-score kernel is +;;; driving the ranking. Events/detections are row hash tables, as storage hands +;;; them back. Run from the repo root with the dylib built and repo on libdirs: +;;; scheme --libdirs $JERBOA/lib --libdirs . --script examples/analytics_check.ss + +(import (jerboa prelude) + (jsecmon analytics)) + +(def fails 0) +(def (check label got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label " => " got + (if ok "" (str " (want " want ")"))))) + +(def (mk-event host sev type ts) + (let ((h (make-hash-table))) + (hash-put! h "host" host) (hash-put! h "severity" sev) + (hash-put! h "event_type" type) (hash-put! h "timestamp_ms" ts) h)) + +(def (mk-anomaly rule host sev ts details) + (let ((h (make-hash-table))) + (hash-put! h "rule" rule) (hash-put! h "host" host) + (hash-put! h "severity" sev) (hash-put! h "timestamp_ms" ts) + (hash-put! h "details" details) h)) + +(def (det1 k v) + (let ((h (make-hash-table))) (hash-put! h k v) h)) + +;; ── risk_prefers_chains_over_noise ───────────────────────────────────────── +;; host-a: 100 info events. host-b: one high event + one critical kill-chain. +(displayln "analytics parity (secmon src/analytics.rs vectors):") +(def events + (append + (map (lambda (i) (mk-event "host-a" "info" "process_start" (+ 1000 i))) + (iota 100)) + (list (mk-event "host-b" "high" "process_start" 2000)))) +(def detections + (list (mk-anomaly "priv_escalation_chain" "host-b" "critical" 2001 (make-hash-table)))) +(def risks (compute-host-risks events detections 10)) + +(check "ranked first host" (host-risk-host (car risks)) "host-b") +(check "host-b ranks above host-a" + (> (host-risk-score (car risks)) (host-risk-score (cadr risks))) #t) +;; host-b: high*3 + 1 distinct rule*6 + 1 chain*25 = 34; host-a: 0 +(check "host-b score" (host-risk-score (car risks)) 34) +(check "host-a score" (host-risk-score (cadr risks)) 0) + +;; ── incidents_collapse_by_key ────────────────────────────────────────────── +(def bf + (list (mk-anomaly "brute_force" "web1" "high" 1000 (det1 "username" "alice")) + (mk-anomaly "brute_force" "web1" "high" 2000 (det1 "username" "alice")) + (mk-anomaly "brute_force" "web1" "high" 3000 (det1 "username" "bob")))) +(def incs (group-incidents bf)) +(check "incident count" (length incs) 2) +(def alice (find (lambda (i) (string=? (incident-key i) "alice")) incs)) +(check "alice occurrences" (incident-occurrences alice) 2) +(check "alice first_ms" (incident-first-ms alice) 1000) +(check "alice last_ms" (incident-last-ms alice) 2000) + +(newline) +(if (= fails 0) + (displayln "OK: untyped analytics matches secmon's vectors.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/analytics.ss @@ -0,0 +1,185 @@ +#!chezscheme +;;; jsecmon analytics — per-host risk ranking and incident grouping, untyped. +;;; +;;; secmon's analytics::compute_host_risks groups already-retrieved events and +;;; detections by host, accumulates signal counts, and ranks hosts by a coarse +;;; 0..100 risk score ("which 5 hosts do I look at first?"). The *scoring* is a +;;; clamped weighted sum — that is the Typed-Jerboa `host-risk-score` kernel. +;;; The grouping / distinct-rule counting / sort / top-N around it is ordinary +;;; Jerboa, which is what this module provides. group_incidents (dedup repeated +;;; detections into incidents) is pure orchestration and lives here too. +;;; +;;; Events and detections are row hash tables (string keys → values), exactly +;;; as the storage layer will hand them back. Verified against secmon's +;;; analytics test vectors (examples/analytics_check.ss). + +(library (jsecmon analytics) + (export compute-host-risks + host-risk? host-risk-host host-risk-score + host-risk-critical host-risk-high host-risk-medium + host-risk-distinct-rules host-risk-chains host-risk-suspicious-cmdline + host-risk-dga host-risk-rootkit-or-tamper host-risk-persistence + host-risk-first-seen-ms host-risk-last-seen-ms + group-incidents + incident? incident-rule incident-host incident-key incident-severity + incident-attack incident-first-ms incident-last-ms + incident-occurrences incident-sample) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + ;; The clamped weighted sum is the typed kernel. Rename so the struct + ;; accessor `host-risk-score` (a result's score) doesn't collide. + (rename (only (jsecmon kernels) host-risk-score) + (host-risk-score risk-kernel))) + + (defstruct host-risk (host score critical high medium distinct-rules chains + suspicious-cmdline dga rootkit-or-tamper persistence + first-seen-ms last-seen-ms)) + (defstruct incident (rule host key severity attack first-ms last-ms + occurrences sample)) + + ;; ── row access (string keys; missing/!typed → default) ───────────────────── + (def (a-str h k) (let ((v (hash-get h k))) (if (string? v) v ""))) + (def (a-num h k d) (let ((v (hash-get h k))) (if (number? v) v d))) + + (def (in? x . opts) (and (member x opts) #t)) + + ;; ── per-host accumulator (a small hash of counters + a rule-name set) ─────── + (def (fresh-accum) + (let ((a (make-hash-table))) + (for-each (lambda (k) (hash-put! a k 0)) + (list "critical" "high" "medium" "chain" "susp" "dga" + "rootkit" "persistence")) + (hash-put! a "first" #f) + (hash-put! a "last" #f) + (hash-put! a "rules" (make-hash-table)) ;; distinct rule names + a)) + (def (bump! a k) (hash-put! a k (+ 1 (hash-get a k)))) + (def (touch! a ts) + (let ((f (hash-get a "first")) (l (hash-get a "last"))) + (hash-put! a "first" (if f (min f ts) ts)) + (hash-put! a "last" (if l (max l ts) ts)))) + + (def (compute-host-risks events detections top-n) + (let ((hosts (make-hash-table))) + (def (accum-for h) + (or (hash-get hosts h) + (let ((a (fresh-accum))) (hash-put! hosts h a) a))) + ;; pass 1 — events: severity tallies, rootkit/persistence, time bounds + (for-each + (lambda (ev) + (let ((a (accum-for (a-str ev "host"))) + (sev (a-str ev "severity")) + (ty (a-str ev "event_type"))) + (cond ((string=? sev "critical") (bump! a "critical")) + ((string=? sev "high") (bump! a "high")) + ((string=? sev "medium") (bump! a "medium"))) + (cond ((in? ty "rootkit_event" "log_tamper_event") (bump! a "rootkit")) + ((string=? ty "persistence_event") (bump! a "persistence"))) + (touch! a (a-num ev "timestamp_ms" 0)))) + events) + ;; pass 2 — detections: distinct rules + chain/cmdline/dga classes + (for-each + (lambda (d) + (let ((a (accum-for (a-str d "host"))) + (rule (a-str d "rule"))) + (hash-put! (hash-get a "rules") rule + (+ 1 (or (hash-get (hash-get a "rules") rule) 0))) + (cond ((in? rule "priv_escalation_chain" "lateral_after_shell" + "persistence_after_access" "log_cover") + (bump! a "chain")) + ((string=? rule "suspicious_cmdline") (bump! a "susp")) + ((string=? rule "dga_domain") (bump! a "dga"))))) + detections) + ;; build, score (typed kernel), rank, truncate + (let* ((rows + (map (lambda (h) + (let* ((a (hash-get hosts h)) + (distinct (length (hash-keys (hash-get a "rules")))) + (score (risk-kernel (hash-get a "critical") + (hash-get a "high") + (hash-get a "medium") + distinct + (hash-get a "chain") + (hash-get a "susp") + (hash-get a "dga") + (hash-get a "rootkit") + (hash-get a "persistence")))) + (make-host-risk h score + (hash-get a "critical") (hash-get a "high") + (hash-get a "medium") distinct (hash-get a "chain") + (hash-get a "susp") (hash-get a "dga") + (hash-get a "rootkit") (hash-get a "persistence") + (hash-get a "first") (hash-get a "last")))) + (hash-keys hosts))) + (ranked (list-sort (lambda (x y) + (if (= (host-risk-score x) (host-risk-score y)) + (string<? (host-risk-host x) (host-risk-host y)) + (> (host-risk-score x) (host-risk-score y)))) + rows))) + (if (> (length ranked) top-n) (take ranked top-n) ranked)))) + + ;; ── incident grouping ─────────────────────────────────────────────────────── + (def (sev-rank s) + (cond ((string=? s "critical") 4) ((string=? s "high") 3) + ((string=? s "medium") 2) ((string=? s "info") 1) (else 0))) + + ;; First non-empty string among `fields` (stringify a non-null non-string). + (def (pick details fields) + (let loop ((fs fields)) + (if (null? fs) + "" + (let ((v (hash-get details (car fs)))) + (cond ((and (string? v) (not (string=? v ""))) v) + ((and v (not (string? v))) (str v)) + (else (loop (cdr fs)))))))) + + (def (dedup-key rule details) + (cond ((string=? rule "brute_force") (pick details '("username"))) + ((string=? rule "credential_stuffing") (pick details '("remote_host"))) + ((in? rule "dns_tunnel" "data_exfil" "recon_port_scan") + (pick details '("process_name"))) + ((string=? rule "suspicious_cron") (pick details '("user"))) + ((string=? rule "suspicious_cmdline") (pick details '("process_name" "exe"))) + ((string=? rule "dga_domain") (pick details '("label" "query_name"))) + ((string=? rule "impossible_travel") (pick details '("username"))) + ((in? rule "priv_escalation_chain" "persistence_after_access" + "log_cover" "lateral_after_shell") "") + (else (pick details '("process_name" "username" "remote_host" + "label" "query_name"))))) + + (def (group-incidents detections) + (let ((groups (make-hash-table))) + (for-each + (lambda (d) + (let* ((rule (a-str d "rule")) (host (a-str d "host")) + (details (or (hash-get d "details") (make-hash-table))) + (key (dedup-key rule details)) + (gk (string-append rule "\x1f;" host "\x1f;" key)) + (sev (a-str d "severity")) (ts (a-num d "timestamp_ms" 0)) + (cur (hash-get groups gk))) + (if cur + (begin + (incident-occurrences-set! cur (+ 1 (incident-occurrences cur))) + (when (< ts (incident-first-ms cur)) (incident-first-ms-set! cur ts)) + (when (> ts (incident-last-ms cur)) (incident-last-ms-set! cur ts)) + (when (> (sev-rank sev) (sev-rank (incident-severity cur))) + (incident-severity-set! cur sev))) + (hash-put! groups gk + (make-incident rule host key sev + (or (hash-get d "attack") '()) ts ts 1 details))))) + detections) + (list-sort (lambda (a b) + (let ((ra (sev-rank (incident-severity a))) + (rb (sev-rank (incident-severity b)))) + (if (= ra rb) + (> (incident-last-ms a) (incident-last-ms b)) + (> ra rb)))) + (hash-values groups)))))