yaml-rules: port secmon's user YAML detection-rule engine (untyped layer)
Jaime Fournier
3fa78f1b6c1b938c0d94db6daf099e95c4ca17d8
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -96,6 +96,11 @@ geoip-check: rust sigma-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/sigma_check.ss +# User-supplied YAML detection rules (threshold/distinct/sequence/match) run +# over a live store, checked against secmon's yaml_rules behaviour. Native lib. +yaml-rules-check: + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/yaml_rules_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -106,6 +111,7 @@ checks: kernels-check $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_store_check.ss $(LOADER_ENV) SECMON_GEOIP_CSV="$(GEOIP_CSV)" $(SCHEME) --libdirs $(LIBDIRS) --script examples/geoip_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/sigma_check.ss + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/yaml_rules_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ make storage-check # SQLite round-trip: store -> query -> detect -> analytics make threats-check # SQL-aggregation threat rules vs secmon detection vectors make geoip-check # geoip CSV vectors + geoip-gated impossible_travel detector make sigma-check # Sigma YAML rule importer vs secmon conversion vectors +make yaml-rules-check # user YAML detection rules (threshold/distinct/sequence/match) make checks # every Jerboa-side check in one shot ``` @@ -75,6 +76,7 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `triage` classifiers | `typed/triage.ss` | ✅ pure predicates (transient-unit?, phantom-rootkit-race?); vectors pass | | `triage` engine (rules + dispatch) | `jsecmon/triage.ss` | ✅ **untyped layer** — all 18 false-positive rules + first-match engine, in secmon's exact RULES order, dispatch in ordinary Jerboa delegating byte/string classification to the typed kernels; 40 triage vectors pass (`make triage-check`), incl. the security-relevant negatives (non-sshd reading host keys, systemd impersonated from /tmp, unknown daemon reading passwd). | | `triage::compute_triaged_ids` (triage-aware mode) | `jsecmon/triage-store.ss` | ✅ **untyped layer** — the bridge above storage+triage: query every in-scope event, triage each, return the sorted benign/expected ID set to drop into a filter's `exclude_event_ids`. `make triage-store-check` proves the round-trip — detection then sees only the real attacks. | +| `storage::yaml_rules` (user rules: threshold/distinct/sequence/match) | `jsecmon/yaml-rules.ss` | ✅ **untyped layer** — port of secmon's `src/storage/yaml_rules.rs`: load YAML rule files (`(std text yaml)`), validate per type, and run them over a `(jsecmon storage)` handle into Anomaly row-hashes. A MatchSpec (event_type/severity/process_name/host + json_eq equality + data_contains LIKE) is inlined into SQL (quotes escaped); window strings (`10m`/`1h`/…) parse to ms; threshold/distinct are time-bucket GROUP BY/HAVING, sequence pairwise-chains steps within the window in Jerboa. `make yaml-rules-check` exercises all four types + window parsing + validation. This is the engine the `sigma` importer feeds. | | `sigma` (Sigma rule importer) | `jsecmon/sigma.ss` | ✅ **untyped layer** — port of secmon's `src/sigma.rs`: parse a Sigma YAML rule (via `(std text yaml)`), map `logsource.category` → event_type, translate the BTreeMap-first selection's fields (`Field` → json_eq, `Field\|contains/startswith/endswith/re` → data_contains, Windows field names aliased to Linux JSON paths), `level` → severity, `attack.tNNNN` tags → ATT&CK IDs, and render secmon's `YamlRule` YAML back out. `make sigma-check` reproduces secmon's four conversion vectors (process_creation, network_connection, unsupported-category skip, safe-name). Pure YAML+strings, untyped. | | `psk::constant_time_eq` | `typed/psk.ss` | ✅ ported, vectors pass | | `psk::from_hex` (hex codec) | `typed/psk.ss` | ✅ hex encode + decode + 32-byte precondition; vectors pass (decode∘encode identity over all 256 byte values) | new file mode 100644 --- /dev/null +++ b/examples/yaml_rules_check.ss @@ -0,0 +1,132 @@ +;;; Parity check for (jsecmon yaml-rules): the four user-rule types (threshold, +;;; distinct, sequence, match) run over a live SQLite store, plus window parsing +;;; and validation. Mirrors secmon's src/storage/yaml_rules.rs behaviour: each +;;; rule is loaded from YAML text, then run to produce Anomaly row-hashes. +;;; +;;; DYLD_LIBRARY_PATH=$JERBOA/lib scheme --libdirs "$JERBOA/lib:." \ +;;; --script examples/yaml_rules_check.ss + +(import (jerboa prelude) + (jsecmon storage) + (jsecmon yaml-rules)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +(def nl (string #\newline)) +(def (lines . xs) (apply string-append (map (lambda (s) (string-append s nl)) xs))) +(def (jdata . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) (json-object->string h) + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) +(def (fresh) (store-open ":memory:")) +(def (rule yaml) (let ((r (parse-yaml-rule yaml))) (if (ok? r) (unwrap r) (error 'rule (unwrap-err r))))) + +;; ── window parsing + validation ────────────────────────────────────────────── +(displayln "window parsing:") +(check "10m" (unwrap (parse-window-ms "10m")) 600000) +(check "30s" (unwrap (parse-window-ms "30s")) 30000) +(check "1h" (unwrap (parse-window-ms "1h")) 3600000) +(check "2d" (unwrap (parse-window-ms "2d")) 172800000) +(check "bare-number is seconds" (unwrap (parse-window-ms "45")) 45000) +(check "empty -> err" (err? (parse-window-ms "")) #t) +(check "bad unit -> err"(err? (parse-window-ms "5x")) #t) + +(displayln "validation:") +(check "threshold w/o threshold -> err" + (err? (parse-yaml-rule (lines "name: r" "type: threshold" "window: 10m" + "match:" " event_type: auth_event"))) + #t) +(check "sequence w/ 1 step -> err" + (err? (parse-yaml-rule (lines "name: r" "type: sequence" "window: 5m" + "steps:" " - event_type: auth_event"))) + #t) +(check "nameless -> err" + (err? (parse-yaml-rule (lines "type: match" "match:" " event_type: x"))) #t) + +;; ── threshold: 5+ failed auths for one user in a 10-min bucket ─────────────── +;; NB: (std text yaml) chokes on double-quoted scalars inside a flow `[...]` +;; sequence, so group_by uses block style (a quoted "$.username" there is fine). +(def brute + (lines "name: brute_force_yaml" "description: 5+ auth failures for same user" + "severity: high" "attack: [T1110.001]" "type: threshold" + "window: 10m" "threshold: 5" + "group_by:" " - host" " - \"$.username\"" + "match:" " event_type: auth_event" " json_eq:" " \"$.success\": 0")) +(displayln "threshold (brute_force_yaml):") +(let ((db (fresh)) (base 1000000)) + (dotimes (k 5) (store-event db (+ 1 k) "h1" "s" (+ base (* k 1000)) "auth_event" "info" #f "sshd" "fail" + (jdata "username" "alice" "success" #f))) + (dotimes (k 2) (store-event db (+ 10 k) "h1" "s" (+ base (* k 1000)) "auth_event" "info" #f "sshd" "fail" + (jdata "username" "bob" "success" #f))) + (let ((r (run-yaml-rule db (rule brute)))) + (check "fires once" (length r) 1) + (check " rule" (hash-get (car r) "rule") "brute_force_yaml") + (check " host" (hash-get (car r) "host") "h1") + (check " severity" (hash-get (car r) "severity") "high") + (check " attack" (hash-get (car r) "attack") '("T1110.001")) + (check " count=5" (hash-get (hash-get (car r) "details") "count") 5) + (check " group=alice" (hash-get (hash-get (car r) "details") "group_values") '("alice"))) + (store-close db)) + +;; ── distinct: 3+ distinct remote ports from a host in a 5-min bucket ───────── +(def portscan + (lines "name: port_fan_yaml" "description: 3+ distinct ports" "severity: medium" + "type: distinct" "window: 5m" "threshold: 3" "distinct: \"$.remote_port\"" + "group_by: [host]" "match:" " event_type: network_connection")) +(displayln "distinct (port_fan_yaml):") +(let ((db (fresh)) (base 1000000)) + (dotimes (k 3) (store-event db (+ 1 k) "h1" "s" (+ base (* k 1000)) "network_connection" "info" #f "nc" "conn" + (jdata "remote_port" (+ 1 k)))) + ;; a second host with only 2 distinct ports -> no fire + (store-event db 20 "h2" "s" base "network_connection" "info" #f "nc" "conn" (jdata "remote_port" 7)) + (store-event db 21 "h2" "s" base "network_connection" "info" #f "nc" "conn" (jdata "remote_port" 7)) + (let ((r (run-yaml-rule db (rule portscan)))) + (check "fires once (h1 only)" (length r) 1) + (check " host" (hash-get (car r) "host") "h1") + (check " dcount=3" (hash-get (hash-get (car r) "details") "distinct_count") 3)) + (store-close db)) + +;; ── sequence: auth success then privilege_escalation within 5 min ─────────── +(def seqrule + (lines "name: priv_esc_yaml" "description: auth success then priv-esc" "severity: critical" + "type: sequence" "window: 5m" + "steps:" " - event_type: auth_event" " json_eq:" " \"$.success\": 1" + " - event_type: privilege_escalation")) +(displayln "sequence (priv_esc_yaml):") +(let ((db (fresh)) (t 1000000)) + (store-event db 1 "h1" "s" t "auth_event" "info" #f "sshd" "ok" (jdata "success" #t)) + (store-event db 2 "h1" "s" (+ t 60000) "privilege_escalation" "high" #f "sudo" "esc" (jdata)) + (let ((r (run-yaml-rule db (rule seqrule)))) + (check "fires once" (length r) 1) + (check " severity" (hash-get (car r) "severity") "critical") + (check " ts=2nd step" (hash-get (car r) "timestamp_ms") (+ t 60000))) + (store-close db)) +(let ((db (fresh)) (t 1000000)) ;; outside the 5-min window -> none + (store-event db 1 "h1" "s" t "auth_event" "info" #f "sshd" "ok" (jdata "success" #t)) + (store-event db 2 "h1" "s" (+ t 600000) "privilege_escalation" "high" #f "sudo" "esc" (jdata)) + (check "seq outside window -> none" (length (run-yaml-rule db (rule seqrule))) 0) + (store-close db)) + +;; ── match: every critical process_start fires once ────────────────────────── +(def matchrule + (lines "name: crit_proc_yaml" "description: critical process" "severity: critical" + "type: match" "match:" " event_type: process_start" " severity: critical")) +(displayln "match (crit_proc_yaml):") +(let ((db (fresh)) (t 1000000)) + (store-event db 1 "h1" "s" t "process_start" "critical" 42 "evil" "ran /tmp/evil" (jdata "exe" "/tmp/evil")) + (store-event db 2 "h1" "s" t "process_start" "info" 43 "ls" "ran ls" (jdata "exe" "/bin/ls")) + (let ((r (run-yaml-rule db (rule matchrule)))) + (check "fires once (critical only)" (length r) 1) + (check " desc has summary" (and (string-contains (hash-get (car r) "description") "ran /tmp/evil") #t) #t)) + (store-close db)) + +(newline) +(if (= fails 0) + (displayln "OK: yaml-rules engine matches secmon's behaviour.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/yaml-rules.ss @@ -0,0 +1,317 @@ +#!chezscheme +;;; jsecmon yaml-rules — user-supplied YAML detection rules, untyped orchestration. +;;; +;;; Port of secmon's src/storage/yaml_rules.rs. Lets operators add detections +;;; without recompiling, in a Sigma-inspired-but-simpler schema with four rule +;;; types covering ~90% of useful detections: +;;; +;;; threshold N+ events matching criteria in a sliding time bucket +;;; distinct N+ DISTINCT values of a field in a bucket +;;; sequence event A then event B within a window on the same host +;;; match every single event matching criteria fires once +;;; +;;; Each rule is loaded from YAML (via (std text yaml)) into a rule hash; a +;;; MatchSpec is a boolean predicate over one event row (event_type / severity / +;;; process_name / host literals plus json_eq equality and data_contains LIKE). +;;; This is pure SQL building + post-processing over a (jsecmon storage) handle, +;;; so it stays untyped. The (jsecmon sigma) importer emits exactly this schema. +;;; +;;; Like the rest of the layer, a rule's own criteria are inlined into the SQL +;;; (string values escaped by doubling quotes, as secmon's escape_sql_string +;;; does) and only the EventFilter clauses are bound params via build-where. +;;; Verified against secmon's yaml_rules behaviour in examples/yaml_rules_check.ss. + +(library (jsecmon yaml-rules) + (export yaml-rule? make-yaml-rule + parse-window-ms validate-rule + parse-yaml-rule load-yaml-rules + run-yaml-rule run-yaml-rules) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (std db sqlite-native) + (std text yaml) + (only (jsecmon storage) make-filter build-where)) + + ;; ── result-row + anomaly helpers (same shapes the rest of the layer uses) ──── + (def (a-of row k) (let ((e (assoc k row))) (and e (cdr e)))) + (def (a-str row k) (let ((v (a-of row k))) (if (string? v) v ""))) + (def (a-num row k) (let ((v (a-of row k))) (if (number? v) v 0))) + (def (val->str v) ;; rusqlite value_to_string analogue + (cond ((eq? v #f) "<null>") ((number? v) (number->string v)) + ((string? v) v) (else (format "~a" v)))) + + (def (details-hash . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) h + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + (def (make-anomaly rule desc host sev ts details attack) + (let ((h (make-hash-table))) + (hash-put! h "rule" rule) (hash-put! h "description" desc) + (hash-put! h "host" host) (hash-put! h "severity" sev) + (hash-put! h "timestamp_ms" ts) (hash-put! h "details" details) + (hash-put! h "attack" attack) h)) + + (def (filtered-query db select-prefix trailing filter) + (let* ((w (build-where filter)) (sql (str select-prefix (car w) trailing))) + (apply sqlite-query db sql (cdr w)))) + + ;; ── a YAML rule + its MatchSpec, as hashes ────────────────────────────────── + (defstruct yaml-rule (name description severity attack type window + group-by threshold distinct match steps)) + + (def (m-ref m key) (let ((e (and (pair? m) (assoc key m)))) (and e (cdr e)))) + (def (mapping-value? v) (and (pair? v) (pair? (car v)) (string? (caar v)))) + (def (s-or m key) (let ((v (m-ref m key))) (and (string? v) v))) + + ;; A MatchSpec hash: event_type/severity/process_name/host (string|#f) + + ;; json_eq/data_contains as (path . value) alists, sorted by path. + (def (sort-alist a) (list-sort (lambda (x y) (string<? (car x) (car y))) a)) + (def (sub-map m key) + (let ((v (m-ref m key))) (if (mapping-value? v) v '()))) + (def (parse-matchspec m) + (let ((h (make-hash-table))) + (hash-put! h "event_type" (s-or m "event_type")) + (hash-put! h "severity" (s-or m "severity")) + (hash-put! h "process_name" (s-or m "process_name")) + (hash-put! h "host" (s-or m "host")) + (hash-put! h "json_eq" (sort-alist (sub-map m "json_eq"))) + (hash-put! h "data_contains" (sort-alist (sub-map m "data_contains"))) + h)) + + ;; ── SQL building for a MatchSpec (inlined, escaped) ────────────────────────── + (def (sql-escape s) ;; double ' like secmon's escape_sql_string + (list->string + (fold-right (lambda (c acc) (if (char=? c #\') (cons #\' (cons #\' acc)) (cons c acc))) + '() (string->list s)))) + (def (sql-quote s) (str "'" (sql-escape s) "'")) + (def (value->sql v) ;; yaml_value_to_sql, inlined as a literal + (cond ((eq? v #t) "1") ((eq? v #f) "0") + ((number? v) (number->string v)) + ((string? v) (sql-quote v)) + ((null? v) "NULL") + (else (sql-quote (format "~a" v))))) + + (def (col-clause spec col sql-col) + (let ((v (hash-get spec col))) (if v (str " AND " sql-col " = " (sql-quote v)) ""))) + (def (match-clauses spec) + (str (col-clause spec "event_type" "event_type") + (col-clause spec "severity" "severity") + (col-clause spec "process_name" "process_name") + (col-clause spec "host" "host") + (apply string-append (map (lambda (kv) + (str " AND json_extract(data, " (sql-quote (car kv)) ") = " + (value->sql (cdr kv)))) + (hash-get spec "json_eq"))) + (apply string-append (map (lambda (kv) + (str " AND json_extract(data, " (sql-quote (car kv)) ") LIKE " + (sql-quote (str "%" (cdr kv) "%")))) + (hash-get spec "data_contains"))))) + + ;; group_by expr -> SQL: a $. path is a json_extract, else a bare column. + (def (group-expr-sql expr) + (if (string-prefix? "$." expr) + (str "json_extract(data, '" (sql-escape expr) "')") + expr)) + (def (group-exprs rule) ;; group_by minus the implicit "host" + (filter (lambda (g) (not (string=? g "host"))) (yaml-rule-group-by rule))) + + ;; ── window parsing ─────────────────────────────────────────────────────────── + (def (parse-window-ms s) ;; -> (ok ms) | (err reason) + (let ((s (string-trim (or s "")))) + (if (string=? s "") (err "empty window") + (let loop ((i 0)) + (if (and (< i (string-length s)) (char-numeric? (string-ref s i))) + (loop (+ i 1)) + (let ((n (string->number (substring s 0 i))) + (unit (substring s i (string-length s)))) + (if (not n) (err (str "invalid duration: " s)) + (let ((mult (cond ((or (string=? unit "") (string=? unit "s")) 1000) + ((string=? unit "m") 60000) + ((string=? unit "h") 3600000) + ((string=? unit "d") 86400000) + (else #f)))) + (if mult (ok (* n mult)) (err (str "unknown unit: " unit))))))))))) + (def (window-ms! rule) + (let ((r (parse-window-ms (yaml-rule-window rule)))) + (if (ok? r) (unwrap r) (error 'yaml-rule (unwrap-err r))))) + + ;; ── parse a YAML document into a yaml-rule ─────────────────────────────────── + (def (parse-yaml-rule yaml-text) ;; -> (ok rule) | (err reason) + (let ((p (try (yaml-load-string yaml-text) (catch (e) #f)))) + (if (not (mapping-value? p)) (err "parse: not a yaml mapping") + (let* ((type (s-or p "type")) + (mc (m-ref p "match")) + (steps-raw (m-ref p "steps")) + (gb (let ((g (m-ref p "group_by"))) (if (list? g) (filter string? g) '()))) + (atk (let ((a (m-ref p "attack"))) (if (list? a) (filter string? a) '()))) + (rule (make-yaml-rule + (or (s-or p "name") "") + (or (s-or p "description") "") + (or (s-or p "severity") "") + atk + (or type "") + (s-or p "window") + gb + (let ((t (m-ref p "threshold"))) (and (number? t) t)) + (s-or p "distinct") + (and (mapping-value? mc) (parse-matchspec mc)) + (if (list? steps-raw) + (map parse-matchspec (filter mapping-value? steps-raw)) + '())))) + (let ((v (validate-rule rule))) + (if (ok? v) (ok rule) v)))))) + + (def (validate-rule rule) ;; -> (ok #t) | (err reason) + (cond + ((string=? (yaml-rule-name rule) "") (err "name is required")) + (else + (let ((ty (yaml-rule-type rule))) + (cond + ((string=? ty "threshold") + (cond ((not (yaml-rule-window rule)) (err "threshold rule needs `window`")) + ((not (yaml-rule-threshold rule)) (err "threshold rule needs `threshold`")) + ((not (yaml-rule-match rule)) (err "threshold rule needs `match`")) + (else (ok #t)))) + ((string=? ty "distinct") + (cond ((not (yaml-rule-window rule)) (err "distinct rule needs `window`")) + ((not (yaml-rule-threshold rule)) (err "distinct rule needs `threshold`")) + ((not (yaml-rule-distinct rule)) (err "distinct rule needs `distinct` field")) + ((not (yaml-rule-match rule)) (err "distinct rule needs `match`")) + (else (ok #t)))) + ((string=? ty "sequence") + (cond ((not (yaml-rule-window rule)) (err "sequence rule needs `window`")) + ((< (length (yaml-rule-steps rule)) 2) (err "sequence rule needs at least 2 steps")) + (else (ok #t)))) + ((string=? ty "match") + (if (not (yaml-rule-match rule)) (err "match rule needs `match`") (ok #t))) + (else (err (str "unknown rule type: '" ty "'")))))))) + + ;; ── load from a file or a directory of .yml/.yaml ──────────────────────────── + (def (yaml-ext? p) (or (string-suffix? ".yml" p) (string-suffix? ".yaml" p))) + (def (load-yaml-rules path) ;; -> list of yaml-rule (warns+skips bad) + (cond + ((file-directory? path) + (let ((files (list-sort string<? + (filter yaml-ext? + (map (lambda (e) (path-join path e)) (directory-list path)))))) + (filter-map + (lambda (p) + (let ((r (try (parse-yaml-rule (read-file-string p)) (catch (e) (err "read error"))))) + (if (ok? r) (unwrap r) + (begin (displayln "warn: skipping " p ": " (unwrap-err r)) #f)))) + files))) + ((file-exists? path) + (let ((r (parse-yaml-rule (read-file-string path)))) + (if (ok? r) (list (unwrap r)) (error 'load-yaml-rules (unwrap-err r))))) + (else (error 'load-yaml-rules (str "no such path: " path))))) + + ;; ── run one rule -> list of anomalies ──────────────────────────────────────── + (def (group-desc rule group-vals) + (let ((ge (group-exprs rule))) + (if (null? group-vals) "" + (str " [" (string-join (map (lambda (g v) (str g "=" v)) ge group-vals) ", ") "]")))) + + (def (run-threshold db rule filter) + (let* ((w (window-ms! rule)) (thr (yaml-rule-threshold rule)) + (ge (group-exprs rule)) (spec (yaml-rule-match rule)) + (gsel (apply string-append (map (lambda (g i) (str ", " (group-expr-sql g) " AS g" i)) ge (iota (length ge))))) + (gby (apply string-append (map (lambda (i) (str ", g" i)) (iota (length ge))))) + (rows (filtered-query db + (str "SELECT host" gsel ", (timestamp_ms / " w ") AS bucket," + " COUNT(*) AS cnt FROM events WHERE 1=1" (match-clauses spec)) + (str " GROUP BY host" gby ", bucket HAVING cnt >= " thr) + filter))) + (map (lambda (r) + (let* ((host (a-str r "host")) (cnt (a-num r "cnt")) (ts (* (a-num r "bucket") w)) + (gv (map (lambda (i) (val->str (a-of r (str "g" i)))) (iota (length ge))))) + (make-anomaly (yaml-rule-name rule) + (str (yaml-rule-description rule) ": " cnt " matches on " host (group-desc rule gv)) + host (yaml-rule-severity rule) ts + (details-hash "count" cnt "threshold" thr "window_ms" w + "group_by" (yaml-rule-group-by rule) "group_values" gv) + (yaml-rule-attack rule)))) + rows))) + + (def (run-distinct db rule filter) + (let* ((w (window-ms! rule)) (thr (yaml-rule-threshold rule)) + (dpath (yaml-rule-distinct rule)) (ge (group-exprs rule)) (spec (yaml-rule-match rule)) + (gsel (apply string-append (map (lambda (g i) (str ", " (group-expr-sql g) " AS g" i)) ge (iota (length ge))))) + (gby (apply string-append (map (lambda (i) (str ", g" i)) (iota (length ge))))) + (rows (filtered-query db + (str "SELECT host" gsel ", (timestamp_ms / " w ") AS bucket," + " COUNT(DISTINCT json_extract(data, '" (sql-escape dpath) "')) AS dcnt" + " FROM events WHERE 1=1" (match-clauses spec)) + (str " GROUP BY host" gby ", bucket HAVING dcnt >= " thr) + filter))) + (map (lambda (r) + (let* ((host (a-str r "host")) (dcnt (a-num r "dcnt")) (ts (* (a-num r "bucket") w)) + (gv (map (lambda (i) (val->str (a-of r (str "g" i)))) (iota (length ge))))) + (make-anomaly (yaml-rule-name rule) + (str (yaml-rule-description rule) ": " dcnt " distinct " dpath " on " host) + host (yaml-rule-severity rule) ts + (details-hash "distinct_count" dcnt "distinct_field" dpath "threshold" thr + "window_ms" w "group_values" gv) + (yaml-rule-attack rule)))) + rows))) + + (def (run-match db rule filter) + (let* ((spec (yaml-rule-match rule)) + (rows (filtered-query db + (str "SELECT host, timestamp_ms, summary FROM events WHERE 1=1" (match-clauses spec)) + "" filter))) + (map (lambda (r) + (let ((host (a-str r "host")) (ts (a-num r "timestamp_ms")) (summary (a-str r "summary"))) + (make-anomaly (yaml-rule-name rule) + (str (yaml-rule-description rule) ": " summary) + host (yaml-rule-severity rule) ts + (details-hash "summary" summary) + (yaml-rule-attack rule)))) + rows))) + + (def (run-sequence db rule filter) + (let ((w (window-ms! rule)) (steps (yaml-rule-steps rule))) + (def (fetch spec) ;; -> list of (host . ts), ordered + (map (lambda (r) (cons (a-str r "host") (a-num r "timestamp_ms"))) + (filtered-query db + (str "SELECT host, timestamp_ms FROM events WHERE 1=1" (match-clauses spec)) + " ORDER BY host, timestamp_ms" filter))) + (let loop ((cands (fetch (car steps))) (rest (cdr steps))) + (if (null? rest) + (map (lambda (c) + (make-anomaly (yaml-rule-name rule) + (str (yaml-rule-description rule) " on " (car c)) + (car c) (yaml-rule-severity rule) (cdr c) + (details-hash "step_count" (length steps) "window_ms" w) + (yaml-rule-attack rule))) + cands) + (let ((next (fetch (car rest)))) + (loop (filter-map + (lambda (c) + (let ((hit (find (lambda (ev) + (and (string=? (car ev) (car c)) + (> (cdr ev) (cdr c)) + (<= (cdr ev) (+ (cdr c) w)))) + next))) + (and hit (cons (car c) (cdr hit))))) + cands) + (cdr rest))))))) + + (def (run-yaml-rule db rule (filter (make-filter))) + (let ((ty (yaml-rule-type rule))) + (cond ((string=? ty "threshold") (run-threshold db rule filter)) + ((string=? ty "distinct") (run-distinct db rule filter)) + ((string=? ty "sequence") (run-sequence db rule filter)) + ((string=? ty "match") (run-match db rule filter)) + (else '())))) + + (def (run-yaml-rules db rules (filter (make-filter))) + (apply append (map (lambda (r) (run-yaml-rule db r filter)) rules))))