threats: time-window aggregates (severity_cluster, off_hours, kill_chain)
Jaime Fournier
6bc39a319f52c661b8eff221337793e324e6ad9b
--- a/README.md +++ b/README.md @@ -80,5 +80,6 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). | | `storage` SQL-aggregation detectors (brute_force, credential_stuffing, dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `run_detections` family: the time-bucket GROUP BY/HAVING rules and the two 5-min sliding-window rules, run as SQL (json_extract) over a `(jsecmon storage)` handle. `make threats-check` reproduces secmon's six detection-rule test vectors. Remaining: the sequence/kill-chain rules (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell, impossible_travel) + frequency/severity/off-hours aggregates. | | `storage` sequence/chain detectors (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_sequence_pair` family: event A then event B within a window on the same host (auth-success→priv-esc /5min, reverse-shell/webshell→persistence /1h, any-critical→log-tampering /1h, shell→lateral /1h). Reproduces secmon's chain test vectors incl. the outside-window negative. | -| `storage` impossible_travel / kill_chain / freq+severity+off-hours aggregates | — | ⏳ impossible_travel needs a geoip CSV loader; kill_chain + frequency/severity/off-hours are the remaining `detect_anomalies` aggregates | +| `storage` time-window aggregates (severity_cluster, off_hours, kill_chain) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_anomalies` family: 5+ crit/high on a host /5min, crit/high outside 08:00-18:00 UTC weekday (SQLite `strftime`), and 3+ distinct kill-chain phases /1h. `run-anomaly-detections` is the dispatcher. `make threats-check` covers each with threshold/negative cases. | +| `storage` frequency_spike + impossible_travel | — | ⏳ frequency_spike needs hourly_counts string-hour bucketing; impossible_travel needs a geoip CSV loader | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | --- a/examples/threats_check.ss +++ b/examples/threats_check.ss @@ -172,6 +172,55 @@ (check "fires once" (length (run-threat-detections db "lateral_after_shell")) 1) (store-close db)) +;; ── severity_cluster: 5+ critical/high on one host in 5min -> 1 ────────────── +;; (no upstream vector; checks the documented threshold semantics) +(displayln "severity_cluster:") +(let ((db (fresh)) (types (list "reverse_shell" "privilege_escalation" "log_tampering" + "lateral_movement" "persistence_event"))) + (let loop ((i 0) (ts types)) + (unless (null? ts) + (store-event db (+ i 1) "h1" "s1" (+ 1000 (* i 1000)) (car ts) "critical" #f #f "e" "{}") + (loop (+ i 1) (cdr ts)))) + (let ((r (run-threat-detections db "severity_cluster"))) + (check "fires once" (length r) 1) + (check " count" (hash-get (hash-get (car r) "details") "count") 5)) + (store-close db)) +(let ((db (fresh))) ;; 4 -> below threshold + (dotimes (i 4) + (store-event db (+ i 1) "h1" "s1" (+ 1000 (* i 1000)) "reverse_shell" "high" #f #f "e" "{}")) + (check "below threshold" (length (run-threat-detections db "severity_cluster")) 0) + (store-close db)) + +;; ── off_hours: critical/high outside 08:00-18:00 UTC weekday ───────────────── +(displayln "off_hours:") +(let ((db (fresh))) + (store-event db 1 "h1" "s1" 1704283200000 "reverse_shell" "high" #f #f "e" "{}") ;; Wed noon -> no + (store-event db 2 "h1" "s1" 1704247200000 "reverse_shell" "high" #f #f "e" "{}") ;; Wed 02:00 -> yes + (store-event db 3 "h1" "s1" 1704542400000 "reverse_shell" "high" #f #f "e" "{}") ;; Sat noon -> yes + (check "fires for night + weekend only" (length (run-threat-detections db "off_hours")) 2) + (store-close db)) + +;; ── kill_chain: 3+ distinct attack phases on one host in 1h -> 1 ───────────── +(displayln "kill_chain:") +(let ((db (fresh))) + (store-event db 1 "h1" "s1" 1000 "dns_query" "info" #f #f "e" "{}") ;; recon + (store-event db 2 "h1" "s1" 2000 "reverse_shell" "critical" #f #f "e" "{}") ;; initial_access + (store-event db 3 "h1" "s1" 3000 "persistence_event" "high" #f #f "e" "{}") ;; persistence + (let ((r (run-threat-detections db "kill_chain"))) + (check "fires once" (length r) 1) + (check " 3 phases" (length (hash-get (hash-get (car r) "details") "phases")) 3)) + (store-close db)) +(let ((db (fresh))) ;; only 2 phases -> 0 + (store-event db 1 "h1" "s1" 1000 "dns_query" "info" #f #f "e" "{}") + (store-event db 2 "h1" "s1" 2000 "reverse_shell" "critical" #f #f "e" "{}") + (check "below phase threshold" (length (run-threat-detections db "kill_chain")) 0) + ;; run-anomaly-detections aggregates the detect_anomalies family in one call + (store-event db 3 "h1" "s1" 3000 "persistence_event" "high" #f #f "e" "{}") + (check "run-anomaly-detections sees kill_chain" + (length (filter (lambda (a) (string=? (hash-get a "rule") "kill_chain")) + (run-anomaly-detections db))) 1) + (store-close db)) + ;; ── run-all dispatch: every rule together, sorted by time ───────────────────── (displayln "run-all + unknown-rule guard:") (let ((db (fresh))) --- a/jsecmon/threats.ss +++ b/jsecmon/threats.ss @@ -27,7 +27,9 @@ detect-brute-force detect-credential-stuffing detect-dns-tunnel detect-suspicious-cron detect-recon-port-scan detect-data-exfil detect-priv-escalation-chain detect-persistence-after-access - detect-log-cover detect-lateral-after-shell) + detect-log-cover detect-lateral-after-shell + detect-severity-cluster detect-off-hours detect-kill-chain + run-anomaly-detections) (import (except (chezscheme) make-hash-table hash-table? sort sort! @@ -268,6 +270,103 @@ (details-hash "trigger_event" (a-str a "event_type") "gap_seconds" (gap-seconds a b))))))) + ;; ── time-window aggregates over the store (secmon detect_anomalies family) ─── + ;; These group by host alone within a window (unlike the per-process slides). + (def (host-window-walk rows window-ms decide) + (let ((v (list->vector rows)) (n (length rows)) (out '())) + (let loop ((i 0)) + (when (< i n) + (let* ((start (vector-ref v i)) + (host (a-str start "host")) + (limit (+ (a-num start "timestamp_ms") window-ms))) + (let scan ((j i)) + (if (and (< j n) + (let ((e (vector-ref v j))) + (and (string=? (a-str e "host") host) + (<= (a-num e "timestamp_ms") limit)))) + (scan (+ j 1)) + (let ((a (decide v i j))) + (cond (a (set! out (cons a out)) (loop j)) + (else (loop (+ i 1)))))))))) + (reverse out))) + + (def (collect-col v i j k) + (let loop ((x i) (acc '())) + (if (>= x j) (reverse acc) + (loop (+ x 1) (cons (a-str (vector-ref v x) k) acc))))) + + ;; 5+ critical/high events on one host inside a 5-min window. + (def (detect-severity-cluster db filter) + (host-window-walk + (filtered-query db + (str "SELECT host, timestamp_ms, event_type, severity FROM events" + " WHERE severity IN ('critical','high')") + " ORDER BY host, timestamp_ms" filter) + window-ms + (lambda (v i j) + (and (>= (- j i) 5) + (let ((start (vector-ref v i))) + (make-anomaly "severity_cluster" (a-str start "host") "high" + (a-num start "timestamp_ms") + (details-hash "count" (- j i) + "event_types" (collect-col v i j "event_type")))))))) + + ;; critical/high events outside 08:00-18:00 UTC on weekdays (weekend = always). + (def (detect-off-hours db filter) + (map (lambda (r) + (make-anomaly "off_hours" (a-str r "host") (a-str r "severity") + (a-num r "timestamp_ms") + (details-hash "event_type" (a-str r "event_type") + "severity" (a-str r "severity")))) + (filtered-query db + (str "SELECT host, timestamp_ms, event_type, severity FROM events" + " WHERE severity IN ('critical','high')" + " AND (CAST(strftime('%w', timestamp_ms/1000, 'unixepoch') AS INTEGER) IN (0,6)" + " OR CAST(strftime('%H', timestamp_ms/1000, 'unixepoch') AS INTEGER) < 8" + " OR CAST(strftime('%H', timestamp_ms/1000, 'unixepoch') AS INTEGER) >= 18)") + " ORDER BY timestamp_ms" filter))) + + ;; Map an event type to a kill-chain phase (or #f). + (def (attack-phase et) + (cond ((or (string=? et "dns_query") (string=? et "network_connection")) "recon") + ((or (string=? et "suspicious_exec") (string=? et "reverse_shell") + (string=? et "webshell")) "initial_access") + ((or (string=? et "persistence_event") (string=? et "scheduled_task_change")) "persistence") + ((string=? et "lateral_movement") "lateral_movement") + ((or (string=? et "privilege_escalation") (string=? et "privilege_change")) "privilege_escalation") + ((string=? et "log_tampering") "defense_evasion") + ((or (string=? et "sensitive_file_access") (string=? et "suspicious_connection")) "exfiltration") + (else #f))) + + ;; 3+ distinct attack phases on one host inside a 1-hour window. + (def (detect-kill-chain db filter) + (host-window-walk + (filtered-query db "SELECT host, timestamp_ms, event_type FROM events WHERE 1=1" + " ORDER BY host, timestamp_ms" filter) + hour-ms + (lambda (v i j) + (let ((phases (make-hash-table)) (types '()) (start (vector-ref v i))) + (let loop ((x i)) + (when (< x j) + (let ((ph (attack-phase (a-str (vector-ref v x) "event_type")))) + (when ph + (hash-put! phases ph #t) + (set! types (cons (a-str (vector-ref v x) "event_type") types)))) + (loop (+ x 1)))) + (and (>= (length (hash-keys phases)) 3) + (make-anomaly "kill_chain" (a-str start "host") "critical" + (a-num start "timestamp_ms") + (details-hash "phases" (hash-keys phases) + "event_types" (reverse types)))))))) + + ;; secmon's detect_anomalies dispatcher (minus frequency_spike, which needs + ;; hourly_counts string-hour bucketing — pending). + (def (run-anomaly-detections db (filter (make-filter))) + (list-sort (lambda (a b) (< (hash-get a "timestamp_ms") (hash-get b "timestamp_ms"))) + (append (detect-severity-cluster db filter) + (detect-off-hours db filter) + (detect-kill-chain db filter)))) + ;; ── dispatch ───────────────────────────────────────────────────────────────── (def threat-rules (list (cons "brute_force" detect-brute-force) @@ -279,7 +378,10 @@ (cons "priv_escalation_chain" detect-priv-escalation-chain) (cons "persistence_after_access" detect-persistence-after-access) (cons "log_cover" detect-log-cover) - (cons "lateral_after_shell" detect-lateral-after-shell))) + (cons "lateral_after_shell" detect-lateral-after-shell) + (cons "severity_cluster" detect-severity-cluster) + (cons "off_hours" detect-off-hours) + (cons "kill_chain" detect-kill-chain))) ;; Run every threat rule (or one by name) over the store, sorted by time — ;; mirrors secmon run_detections(rule_name, filter). filter defaults to all.