threats: SQL-aggregation detection rules (jsecmon threats)
Jaime Fournier <jaimef@linbsd.org>
e4d33f0e0930fce6a7ef556628dd9f1d524dff34
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check analytics-check detect-check storage-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check analytics-check detect-check storage-check threats-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -69,12 +69,20 @@ storage-check: rust cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss +# The SQL-aggregation threat detectors (brute_force, credential_stuffing, +# dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) over a live store, +# checked against secmon's run_detections test vectors. Needs the native lib too. +threats-check: rust + cd $(BUILD) && cargo build --release + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/threats_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/threats_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ make triage-check # verify the untyped (jsecmon triage) engine vs secmon vect make analytics-check # verify untyped risk-ranking + incident grouping vs vectors make detect-check # full pipeline: events -> detect -> analytics (all kernels) make storage-check # SQLite round-trip: store -> query -> detect -> analytics +make threats-check # SQL-aggregation threat rules vs secmon detection vectors make checks # every Jerboa-side check in one shot ``` @@ -77,5 +78,6 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `psk` HKDF/SHA256/AES-GCM | — | ⏳ FFI-delegated to vetted crates (not reimplemented) | | `crypto::ecies` | — | ⏳ FFI-delegated; orchestration only | | `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). | -| `storage` SQL-aggregation detectors (brute_force, dns_tunnel, …) | — | ⏳ next: GROUP BY/window detection rules over the store | +| `storage` SQL-aggregation detectors (brute_force, credential_stuffing, dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `run_detections` family: the time-bucket GROUP BY/HAVING rules and the two 5-min sliding-window rules, run as SQL (json_extract) over a `(jsecmon storage)` handle. `make threats-check` reproduces secmon's six detection-rule test vectors. Remaining: the sequence/kill-chain rules (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell, impossible_travel) + frequency/severity/off-hours aggregates. | +| `storage` sequence/chain detectors (priv_escalation, kill_chain, impossible_travel, …) | — | ⏳ next: sequence-pair + multi-phase window rules | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | new file mode 100644 --- /dev/null +++ b/examples/threats_check.ss @@ -0,0 +1,144 @@ +;;; Threat-rule check: the SQL-aggregation detectors over a live store. +;;; +;;; Reproduces secmon's run_detections test vectors (src/storage/mod.rs +;;; #[test] detect_*): each rule's positive case fires exactly one anomaly, and +;;; brute_force's below-threshold case fires none. Events are written through +;;; (jsecmon storage); detection runs over the SQLite store via (jsecmon +;;; threats), proving the time-bucket GROUP BY rules and the two sliding-window +;;; rules port faithfully (json_extract / integer-bucket math / window walk). +;;; +;;; Run from the repo root with the dylib built and repo on libdirs: +;;; scheme --libdirs "$JERBOA/lib:." --script examples/threats_check.ss + +(import (jerboa prelude) + (jsecmon storage) + (jsecmon threats)) + +(def fails 0) +(def (check label got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label " => " got + (if ok "" (str " (want " want ")"))))) + +;; data column as secmon writes it: a JSON string. `success` is a JSON boolean, +;; so json_extract(data,'$.success') yields 1/0 — the SQL matches it against 0. +(def (jdata . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) (json-object->string h) + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + +(def (fresh) (store-open ":memory:")) + +;; sanity: a JSON boolean false must extract as 0 (the whole auth family hinges +;; on this — if #f serialized as null/elsewise, json_extract(...)=0 would miss) +(displayln "json boolean serialization:") +(let ((db (fresh))) + (store-event db 1 "h" "s" 1000 "auth_event" "info" #f #f "a" (jdata "success" #f)) + (check "success:false extracts as 0" + (length (query-events db (make-filter "search" "false"))) 1) + (store-close db)) + +;; ── brute_force: 6 failed auths for one user in one 10-min bucket -> 1 ──────── +(displayln "brute_force:") +(let ((db (fresh))) + (dotimes (i 6) + (store-event db (+ i 1) "h1" "s1" (+ 600000 (* i 1000)) "auth_event" "info" + #f #f "auth" (jdata "username" "admin" "success" #f "auth_type" "ssh"))) + (let ((r (run-threat-detections db "brute_force"))) + (check "fires once" (length r) 1) + (check " rule" (hash-get (car r) "rule") "brute_force") + (check " severity" (hash-get (car r) "severity") "high") + (check " failure_count" (hash-get (hash-get (car r) "details") "failure_count") 6)) + (store-close db)) + +;; below threshold: 4 failures -> 0 +(let ((db (fresh))) + (dotimes (i 4) + (store-event db (+ i 1) "h1" "s1" (+ 600000 (* i 1000)) "auth_event" "info" + #f #f "auth" (jdata "username" "admin" "success" #f))) + (check "brute_force below threshold" (length (run-threat-detections db "brute_force")) 0) + (store-close db)) + +;; ── credential_stuffing: 5 distinct users from one remote_host /10min -> 1 ──── +(displayln "credential_stuffing:") +(let ((db (fresh)) (users (list "alice" "bob" "charlie" "dave" "eve"))) + (let loop ((i 0) (us users)) + (unless (null? us) + (store-event db (+ i 1) "h1" "s1" (+ 600000 (* i 1000)) "auth_event" "info" + #f #f "auth" (jdata "username" (car us) "success" #f "remote_host" "10.0.0.99")) + (loop (+ i 1) (cdr us)))) + (let ((r (run-threat-detections db "credential_stuffing"))) + (check "fires once" (length r) 1) + (check " distinct_usernames" (hash-get (hash-get (car r) "details") "distinct_usernames") 5)) + (store-close db)) + +;; ── dns_tunnel: 55 dns queries from one process in one 5-min bucket -> 1 ────── +(displayln "dns_tunnel:") +(let ((db (fresh))) + (dotimes (i 55) + (store-event db (+ i 1) "h1" "s1" (+ 300000 (* i 100)) "dns_query" "info" + #f "dnscat" "dns" (jdata "query_name" (str i ".tunnel.evil.com") "query_type" "TXT"))) + (let ((r (run-threat-detections db "dns_tunnel"))) + (check "fires once" (length r) 1) + (check " query_count" (hash-get (hash-get (car r) "details") "query_count") 55)) + (store-close db)) + +;; ── suspicious_cron: non-root cron change fires, root does not -> 1 ─────────── +(displayln "suspicious_cron:") +(let ((db (fresh))) + (store-event db 1 "h1" "s1" 1000 "scheduled_task_change" "medium" #f #f "cron" + (jdata "user" "www-data" "path" "/etc/crontab")) + (store-event db 2 "h1" "s1" 2000 "scheduled_task_change" "medium" #f #f "cron" + (jdata "user" "root" "path" "/etc/crontab")) + (let ((r (run-threat-detections db "suspicious_cron"))) + (check "fires once (non-root only)" (length r) 1) + (check " user" (hash-get (hash-get (car r) "details") "user") "www-data")) + (store-close db)) + +;; ── recon_port_scan: one process -> 12 distinct ports in 5min -> 1 ──────────── +(displayln "recon_port_scan:") +(let ((db (fresh))) + (dotimes (i 12) + (store-event db (+ i 1) "h1" "s1" (+ 1000 (* i 1000)) "network_connection" "info" + #f "nmap" "conn" (jdata "remote_addr" "10.0.0.1" "remote_port" (+ 1000 i)))) + (let ((r (run-threat-detections db "recon_port_scan"))) + (check "fires once" (length r) 1) + (check " distinct_ports" (hash-get (hash-get (car r) "details") "distinct_ports") 12)) + (store-close db)) + +;; ── data_exfil: one process -> 22 outbound connections in 5min -> 1 ─────────── +(displayln "data_exfil:") +(let ((db (fresh))) + (dotimes (i 22) + (store-event db (+ i 1) "h1" "s1" (+ 1000 (* i 100)) "network_connection" "info" + #f "curl" "conn" (jdata "remote_addr" (str "10.0.0." (modulo i 10)) "remote_port" 443))) + (let ((r (run-threat-detections db "data_exfil"))) + (check "fires once" (length r) 1) + (check " connection_count" (hash-get (hash-get (car r) "details") "connection_count") 22)) + (store-close db)) + +;; ── run-all dispatch: every rule together, sorted by time ───────────────────── +(displayln "run-all + unknown-rule guard:") +(let ((db (fresh))) + (dotimes (i 6) + (store-event db (+ i 1) "h1" "s1" (+ 600000 (* i 1000)) "auth_event" "info" + #f #f "auth" (jdata "username" "admin" "success" #f))) + (dotimes (i 55) + (store-event db (+ i 100) "h1" "s1" (+ 300000 (* i 100)) "dns_query" "info" + #f "dnscat" "dns" (jdata "query_name" (str i ".x.evil.com")))) + (let ((r (run-threat-detections db))) + ;; dns_tunnel bucket ts=300000 sorts before brute_force bucket ts=600000 + (check "all rules -> 2 anomalies" (length r) 2) + (check " sorted by time (dns first)" (hash-get (car r) "rule") "dns_tunnel")) + (check "unknown rule errors" + (try (begin (run-threat-detections db "nope") "no-error") + (catch (e) "errored")) + "errored") + (store-close db)) + +(newline) +(if (= fails 0) + (displayln "OK: SQL-aggregation threat rules match secmon's detection vectors.") + (begin (displayln fails " FAILURES") (exit 1))) --- a/jsecmon/storage.ss +++ b/jsecmon/storage.ss @@ -21,7 +21,7 @@ (library (jsecmon storage) (export store-open store-close store-event store-count - make-filter query-events) + make-filter query-events build-where) (import (except (chezscheme) make-hash-table hash-table? sort sort! new file mode 100644 --- /dev/null +++ b/jsecmon/threats.ss @@ -0,0 +1,214 @@ +#!chezscheme +;;; jsecmon threats — the SQL-aggregation detection rules, untyped orchestration. +;;; +;;; secmon's src/storage/mod.rs run_detections family: stateful threat rules that +;;; aggregate over the event store (GROUP BY / time-bucket / sliding window) +;;; rather than scoring one event at a time. They are pure SQL + post-processing, +;;; so they live in the untyped layer and run directly on a (jsecmon storage) +;;; handle. (The per-event scoring rules — suspicious_cmdline, dga_domain — are +;;; in (jsecmon detect), driven by the typed kernels.) +;;; +;;; Six rules ported here, faithful to secmon's thresholds and SQL: +;;; brute_force 5+ failed auths for one username in a 10-min bucket +;;; credential_stuffing 5+ distinct usernames failing from one remote_host /10min +;;; dns_tunnel 50+ dns_query from one process in a 5-min bucket +;;; suspicious_cron scheduled_task_change by a non-root user +;;; recon_port_scan one process -> 10+ distinct remote ports in a 5-min slide +;;; data_exfil one process -> 20+ outbound connections in a 5-min slide +;;; +;;; The bucket rules are GROUP BY ... HAVING; the two sliding-window rules pull +;;; ordered rows and walk them in Jerboa exactly as secmon walks its Vec. Uses +;;; json_extract (json1 is present in the rusqlite this binds). Anomalies are the +;;; same row-hash shape (jsecmon analytics) consumes. Verified in +;;; examples/threats_check.ss against secmon's own detection-rule test vectors. + +(library (jsecmon threats) + (export run-threat-detections threat-rules + detect-brute-force detect-credential-stuffing detect-dns-tunnel + detect-suspicious-cron detect-recon-port-scan detect-data-exfil) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (std db sqlite-native) + (only (jsecmon storage) make-filter build-where)) + + ;; ── anomaly construction (row hash, same shape detect/analytics use) ──────── + (def (details-hash . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) + h + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + (def (make-anomaly rule host sev ts details) + (let ((h (make-hash-table))) + (hash-put! h "rule" rule) (hash-put! h "host" host) + (hash-put! h "severity" sev) (hash-put! h "timestamp_ms" ts) + (hash-put! h "details" details) (hash-put! h "attack" '()) + h)) + + (def (a-of row k) (cdr (assoc k row))) ;; column value from a result alist + (def (a-str row k) (let ((v (a-of row k))) (if (string? v) v ""))) + (def (a-num row k) (let ((v (a-of row k))) (if (number? v) v 0))) + + ;; Run a SQL query that already embeds its fixed conditions, appending the + ;; EventFilter clauses (and their binds) after them, then the trailing clause. + (def (filtered-query db select-prefix trailing filter) + (let* ((w (build-where filter)) + (sql (str select-prefix (car w) trailing))) + (apply sqlite-query db sql (cdr w)))) + + ;; ── time-bucket GROUP BY rules ────────────────────────────────────────────── + (def (detect-brute-force db filter) + (map (lambda (r) + (let ((ts (* (a-num r "time_bucket") 600000))) + (make-anomaly "brute_force" (a-str r "host") "high" ts + (details-hash "username" (a-str r "username") + "failure_count" (a-num r "cnt") + "window_start_ms" ts)))) + (filtered-query db + (str "SELECT host, json_extract(data,'$.username') AS username," + " (timestamp_ms/600000) AS time_bucket, COUNT(*) AS cnt" + " FROM events WHERE event_type='auth_event'" + " AND json_extract(data,'$.success')=0") + " GROUP BY host, username, time_bucket HAVING cnt>=5" + filter))) + + (def (detect-credential-stuffing db filter) + (map (lambda (r) + (let ((ts (* (a-num r "time_bucket") 600000))) + (make-anomaly "credential_stuffing" (a-str r "host") "high" ts + (details-hash "remote_host" (a-str r "remote_host") + "distinct_usernames" (a-num r "user_cnt") + "window_start_ms" ts)))) + (filtered-query db + (str "SELECT host, json_extract(data,'$.remote_host') AS remote_host," + " (timestamp_ms/600000) AS time_bucket," + " COUNT(DISTINCT json_extract(data,'$.username')) AS user_cnt" + " FROM events WHERE event_type='auth_event'" + " AND json_extract(data,'$.success')=0" + " AND json_extract(data,'$.remote_host') IS NOT NULL") + " GROUP BY host, remote_host, time_bucket HAVING user_cnt>=5" + filter))) + + (def (detect-dns-tunnel db filter) + (map (lambda (r) + (let ((ts (* (a-num r "time_bucket") 300000))) + (make-anomaly "dns_tunnel" (a-str r "host") "high" ts + (details-hash "process_name" (a-str r "process_name") + "query_count" (a-num r "cnt") + "window_start_ms" ts)))) + (filtered-query db + (str "SELECT host, process_name, (timestamp_ms/300000) AS time_bucket," + " COUNT(*) AS cnt FROM events" + " WHERE event_type='dns_query' AND process_name IS NOT NULL") + " GROUP BY host, process_name, time_bucket HAVING cnt>=50" + filter))) + + ;; ── per-row rule ───────────────────────────────────────────────────────────── + (def (detect-suspicious-cron db filter) + (filter-map + (lambda (r) + (let* ((data-str (a-str r "data")) + (data (if (string=? data-str "") (make-hash-table) + (try (string->json-object data-str) + (catch (e) (make-hash-table))))) + (user (let ((u (hash-get data "user"))) (if (string? u) u "")))) + (and (not (string=? user "root")) + (make-anomaly "suspicious_cron" (a-str r "host") "medium" + (a-num r "timestamp_ms") + (details-hash "user" user "summary" (a-str r "summary")))))) + (filtered-query db + (str "SELECT host, timestamp_ms, summary, data FROM events" + " WHERE event_type='scheduled_task_change'") + "" + filter))) + + ;; ── sliding-window rules (walk ordered rows exactly as secmon walks its Vec) ── + (def window-ms 300000) ;; 5 minutes + + ;; Generic 5-min slide over rows ordered by (host, process_name, timestamp_ms). + ;; For each maximal run sharing host+process within the window, `decide` is + ;; called with (rows-vector start-index end-index) and returns an anomaly or #f. + (def (slide-detect rows decide) + (let ((v (list->vector rows)) (n (length rows)) (out '())) + (let loop ((i 0)) + (when (< i n) + (let* ((start (vector-ref v i)) + (host (a-str start "host")) + (pname (a-str start "process_name")) + (limit (+ (a-num start "timestamp_ms") window-ms))) + (let scan ((j i)) + (if (and (< j n) + (let ((e (vector-ref v j))) + (and (string=? (a-str e "host") host) + (string=? (a-str e "process_name") pname) + (<= (a-num e "timestamp_ms") limit)))) + (scan (+ j 1)) + (let ((a (decide v i j))) + (cond (a (set! out (cons a out)) (loop j)) + (else (loop (+ i 1)))))))))) + (reverse out))) + + (def (detect-recon-port-scan db filter) + (slide-detect + (filtered-query db + (str "SELECT host, timestamp_ms, process_name," + " json_extract(data,'$.remote_addr') AS remote_addr," + " CAST(json_extract(data,'$.remote_port') AS TEXT) AS remote_port" + " FROM events WHERE event_type='network_connection'" + " AND process_name IS NOT NULL") + " ORDER BY host, process_name, timestamp_ms" + filter) + (lambda (v i j) + (let ((ports (make-hash-table)) (start (vector-ref v i))) + (let count ((k i)) (when (< k j) + (hash-put! ports (a-str (vector-ref v k) "remote_port") #t) (count (+ k 1)))) + (let ((distinct (length (hash-keys ports)))) + (and (>= distinct 10) + (make-anomaly "recon_port_scan" (a-str start "host") "medium" + (a-num start "timestamp_ms") + (details-hash "process_name" (a-str start "process_name") + "distinct_ports" distinct + "window_start_ms" (a-num start "timestamp_ms"))))))))) + + (def (detect-data-exfil db filter) + (slide-detect + (filtered-query db + (str "SELECT host, timestamp_ms, process_name FROM events" + " WHERE event_type='network_connection' AND process_name IS NOT NULL") + " ORDER BY host, process_name, timestamp_ms" + filter) + (lambda (v i j) + (let ((count (- j i)) (start (vector-ref v i))) + (and (>= count 20) + (make-anomaly "data_exfil" (a-str start "host") "high" + (a-num start "timestamp_ms") + (details-hash "process_name" (a-str start "process_name") + "connection_count" count + "window_start_ms" (a-num start "timestamp_ms")))))))) + + ;; ── dispatch ───────────────────────────────────────────────────────────────── + (def threat-rules + (list (cons "brute_force" detect-brute-force) + (cons "credential_stuffing" detect-credential-stuffing) + (cons "dns_tunnel" detect-dns-tunnel) + (cons "suspicious_cron" detect-suspicious-cron) + (cons "recon_port_scan" detect-recon-port-scan) + (cons "data_exfil" detect-data-exfil))) + + ;; Run every threat rule (or one by name) over the store, sorted by time — + ;; mirrors secmon run_detections(rule_name, filter). filter defaults to all. + (def (run-threat-detections db (rule #f) (filter (make-filter))) + (let ((rules (if rule + (let ((p (assoc rule threat-rules))) + (if p (list p) (error 'run-threat-detections "unknown rule" rule))) + threat-rules))) + (list-sort (lambda (a b) (< (hash-get a "timestamp_ms") (hash-get b "timestamp_ms"))) + (append-map (lambda (p) ((cdr p) db filter)) rules)))))