threats: port frequency_spike, completing the detect_anomalies family
Jaime Fournier <jaimef@linbsd.org>
e60c4fe9241fc85690c62c7deda9efb419baecee
diff --git a/README.md b/README.md
index 16c085f..d175bd2 100644
--- a/README.md
+++ b/README.md
@@ -81,6 +81,6 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage).
| `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). |
| `storage` SQL-aggregation detectors (brute_force, credential_stuffing, dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `run_detections` family: the time-bucket GROUP BY/HAVING rules and the two 5-min sliding-window rules, run as SQL (json_extract) over a `(jsecmon storage)` handle. `make threats-check` reproduces secmon's six detection-rule test vectors. Remaining: the sequence/kill-chain rules (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell, impossible_travel) + frequency/severity/off-hours aggregates. |
| `storage` sequence/chain detectors (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_sequence_pair` family: event A then event B within a window on the same host (auth-success→priv-esc /5min, reverse-shell/webshell→persistence /1h, any-critical→log-tampering /1h, shell→lateral /1h). Reproduces secmon's chain test vectors incl. the outside-window negative. |
-| `storage` time-window aggregates (severity_cluster, off_hours, kill_chain) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_anomalies` family: 5+ crit/high on a host /5min, crit/high outside 08:00-18:00 UTC weekday (SQLite `strftime`), and 3+ distinct kill-chain phases /1h. `run-anomaly-detections` is the dispatcher. `make threats-check` covers each with threshold/negative cases. |
-| `storage` frequency_spike + impossible_travel | — | ⏳ frequency_spike needs hourly_counts string-hour bucketing; impossible_travel needs a geoip CSV loader |
+| `storage` time-window aggregates (frequency_spike, severity_cluster, off_hours, kill_chain) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's full `detect_anomalies` family: per-(host,event_type) hour count 3x above its own average, 5+ crit/high on a host /5min, crit/high outside 08:00-18:00 UTC weekday (SQLite `strftime`), and 3+ distinct kill-chain phases /1h. `run-anomaly-detections` is the dispatcher (frequency_spike first, as secmon runs it). `make threats-check` covers each with threshold/negative cases. |
+| `storage` impossible_travel | — | ⏳ needs a geoip CSV loader (IP→country); secmon gates it behind geoip availability |
| monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last |
diff --git a/examples/threats_check.ss b/examples/threats_check.ss
index 680b79c..ffd7a5a 100644
--- a/examples/threats_check.ss
+++ b/examples/threats_check.ss
@@ -221,6 +221,31 @@
(run-anomaly-detections db))) 1)
(store-close db))
+;; ── frequency_spike: an hour 3x above the (host,event_type) average -> 1 ──────
+(displayln "frequency_spike:")
+(let ((db (fresh)) (base 1704067200000)) ;; 2024-01-01 00:00:00 UTC, hour-aligned
+ (store-event db 1 "h1" "s1" base "process_start" "info" #f "p" "e" "{}")
+ (store-event db 2 "h1" "s1" (+ base 3600000) "process_start" "info" #f "p" "e" "{}")
+ (store-event db 3 "h1" "s1" (+ base 7200000) "process_start" "info" #f "p" "e" "{}")
+ (dotimes (i 10) ;; 10 in the 4th hour -> 13/4 avg 3.25, 10>9.75
+ (store-event db (+ 10 i) "h1" "s1" (+ base 10800000 (* i 1000))
+ "process_start" "info" #f "p" "e" "{}"))
+ (let ((r (detect-frequency-spike db (make-filter))))
+ (check "fires once" (length r) 1)
+ (check " rule" (hash-get (car r) "rule") "frequency_spike")
+ (check " severity" (hash-get (car r) "severity") "medium")
+ (check " count=10" (hash-get (hash-get (car r) "details") "count") 10)
+ (check " ts=4th hour" (hash-get (car r) "timestamp_ms") (+ base 10800000)))
+ (check "run-anomaly-detections sees frequency_spike"
+ (length (filter (lambda (a) (string=? (hash-get a "rule") "frequency_spike"))
+ (run-anomaly-detections db))) 1)
+ (store-close db))
+(let ((db (fresh)) (base 1704067200000)) ;; flat 1/hr -> avg 1, never 3x -> 0
+ (dotimes (i 4)
+ (store-event db (+ 1 i) "h1" "s1" (+ base (* i 3600000)) "process_start" "info" #f "p" "e" "{}"))
+ (check "frequency_spike flat -> none" (length (detect-frequency-spike db (make-filter))) 0)
+ (store-close db))
+
;; ── run-all dispatch: every rule together, sorted by time ─────────────────────
(displayln "run-all + unknown-rule guard:")
(let ((db (fresh)))
diff --git a/jsecmon/threats.ss b/jsecmon/threats.ss
index 1db0374..482d9a0 100644
--- a/jsecmon/threats.ss
+++ b/jsecmon/threats.ss
@@ -28,6 +28,7 @@
detect-suspicious-cron detect-recon-port-scan detect-data-exfil
detect-priv-escalation-chain detect-persistence-after-access
detect-log-cover detect-lateral-after-shell
+ detect-frequency-spike
detect-severity-cluster detect-off-hours detect-kill-chain
run-anomaly-detections)
(import (except (chezscheme)
@@ -359,11 +360,44 @@
(details-hash "phases" (hash-keys phases)
"event_types" (reverse types))))))))
- ;; secmon's detect_anomalies dispatcher (minus frequency_spike, which needs
- ;; hourly_counts string-hour bucketing — pending).
+ ;; frequency_spike: per (host,event_type), flag any hour whose event count
+ ;; exceeds 3x that pair's average hourly count. secmon buckets via
+ ;; strftime('%Y-%m-%d %H:00', ...) then parse_hour_to_ms; both are UTC, so the
+ ;; bucket start in ms is just floor(ts/1h)*1h — computed directly in SQL here.
+ (def (detect-frequency-spike db (filter (make-filter)))
+ (let ((rows (filtered-query db
+ (str "SELECT host, event_type,"
+ " (timestamp_ms/3600000)*3600000 AS hour_ms, COUNT(*) AS cnt"
+ " FROM events WHERE 1=1")
+ " GROUP BY host, event_type, hour_ms ORDER BY hour_ms" filter)))
+ (if (null? rows)
+ '()
+ (let ((totals (make-hash-table))) ;; (host event_type) -> (cons sum hours)
+ (for-each
+ (lambda (r)
+ (let* ((k (list (a-str r "host") (a-str r "event_type")))
+ (cnt (a-num r "cnt"))
+ (cur (hash-get totals k)))
+ (hash-put! totals k
+ (if cur (cons (+ (car cur) cnt) (+ (cdr cur) 1)) (cons cnt 1)))))
+ rows)
+ (filter-map
+ (lambda (r)
+ (let* ((host (a-str r "host")) (et (a-str r "event_type"))
+ (cnt (a-num r "cnt"))
+ (agg (hash-get totals (list host et)))
+ (avg (/ (exact->inexact (car agg)) (cdr agg))))
+ (and (> avg 0.0) (> (exact->inexact cnt) (* avg 3.0))
+ (make-anomaly "frequency_spike" host "medium" (a-num r "hour_ms")
+ (details-hash "event_type" et "count" cnt
+ "average" avg "ratio" (/ (exact->inexact cnt) avg))))))
+ rows)))))
+
+ ;; secmon's detect_anomalies dispatcher, in its run order (frequency_spike first).
(def (run-anomaly-detections db (filter (make-filter)))
(list-sort (lambda (a b) (< (hash-get a "timestamp_ms") (hash-get b "timestamp_ms")))
- (append (detect-severity-cluster db filter)
+ (append (detect-frequency-spike db filter)
+ (detect-severity-cluster db filter)
(detect-off-hours db filter)
(detect-kill-chain db filter))))