correlate: port storage detect_* GROUP-BY rules as pure cores (batch 1)
ober
865931ee90a45b656c9f42517dac258d08e69763
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -258,6 +258,12 @@ ioc-check: frame-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/frame_check.ss +# Cross-event correlation rules (secmon storage/mod.rs detect_*), pure cores over +# pre-shaped rows: brute_force/credential_stuffing (≥5 / 10min), dns_tunnel +# (≥50 / 5min), suspicious_cron (non-root). SQL fetch + format_ts are deferred. +correlate-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/correlate_check.ss + # Reverse-shell classifiers (secmon src/monitor/revshell.rs): is_shell / # is_revshell_tool (exact name), is_c2_port (17-port set), is_legitimate_service # (443/8080/8443 + name substring), extract_addr_from_cmdline (/dev/tcp/ split), @@ -337,6 +343,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/ioc_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/frame_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/correlate_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/revshell_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/cron_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/logtamper_check.ss --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ make collector-cli-check # collector bin: --after/--format/--db + host normalize make event-summary-check # storage readers: extract_pid/extract_process_name/build_summary make ioc-check # storage IOC: detect_ioc_type + is_ipv4 + parse_ioc_text + match field routing make frame-check # server protocol: length-prefixed wire framing (encode/read_length/decode) +make correlate-check # storage detect_*: brute_force/cred_stuffing/dns_tunnel/suspicious_cron cores make revshell-check # revshell: is_shell/is_c2_port/is_legitimate + classify_connection make cron-check # cron: per-platform CRON/PERIODIC path tables + systemd/periodic route make logtamper-check # logtamper: system-log/history tables + classify-tamper (trunc/mtime) @@ -127,6 +128,7 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `geoip` (CSV GeoIP/ASN, IPv4+IPv6, binary-search range lookup, is_private) | `jsecmon/geoip.ss` | ✅ **untyped layer** — full port of secmon's `src/geoip.rs`: parse `start,end,country,asn,name` CSV rows (v4 + one-`::`-expanding v6), sort-by-start + binary-search lookup, RFC1918/loopback/link-local/multicast/ULA/CGNAT → synthetic `PRIVATE`. Pure parsing + integer math + file read, so untyped. `make geoip-check` runs secmon's geoip vectors. | | `storage` impossible_travel | `jsecmon/threats.ss` | ✅ **untyped layer** — geoip-gated (reads `SECMON_GEOIP_CSV`): pair a user's consecutive successful `auth_event`s, fire `high` when the two source IPs resolve to different countries within `SECMON_TRAVEL_GAP_MIN` (default 30). Private IPs are dropped before pairing. `make geoip-check` proves the fire + the gap/same-country/private/cross-user negatives. | | `buffer::ring` (StoredEvent ring buffer) | `jsecmon/buffer.ss` | ✅ **untyped layer** — port of secmon's `src/buffer/ring.rs`: the agent's bounded in-memory event ring. FIFO list + monotonic seq numbering, priority eviction (`event_severity_u8` table, drop lowest-severity oldest-first, oldest-critical last), seq/time-range polling, FIFO delivery-ack (`clear_before`), and the little-endian header codec (`seq u64 ∥ ts i64 ∥ sev u8 ∥ payload`). Pure mechanics, so untyped — the one security step, ECIES payload encryption, is FFI-deferred: the caller hands `buffer-store!` opaque ciphertext bytes. `make buffer-check` reproduces secmon's three ring tests (store/seq, priority eviction, FIFO-oldest) + codec round-trip. | +| `storage` correlation rules (`detect_*`) | `jsecmon/correlate.ss` | ✅ **untyped layer** — secmon's anomaly detectors are "run a SQL query, then map result rows → Anomaly". The SQL fetch (with its `json_extract` / `GROUP BY … HAVING`) is the deferred storage I/O; the pure, portable part is the correlation algorithm over the fetched rows + the Anomaly description, reimplemented here over pre-shaped row lists (which is exactly what `test_detect_*` exercise). **Batch 1** = the GROUP-BY rules: `detect_brute_force` (≥5 failed-auth per user / 10-min bucket), `detect_credential_stuffing` (≥5 **distinct** users per remote / 10-min), `detect_dns_tunnel` (≥50 queries per process / 5-min), `detect_suspicious_cron` (per-row: `scheduled_task_change` by a non-root user, missing user → `""` ≠ root → kept). `timestamp_ms / N` is integer division → `quotient`, bucket anomaly ts is `bucket*N`; COUNT(*) vs COUNT(DISTINCT) become first-seen-order tallies (treat the set, not order, as significant). Each detector returns "anomaly seeds" (an Anomaly minus the calendar `format_ts` fields, which the caller adds, and minus the `attack` tags from `anomaly-rule-attack`). `make correlate-check` reproduces those four tests + the threshold/bucket-boundary/distinct-dedup edges. | | `server::protocol` (length-prefixed framing) | `jsecmon/frame.ss` | ✅ **untyped layer** — the wire framing the collector↔agent transport wraps every `ProtocolMessage` in: `frame_encode` prepends a 4-byte little-endian u32 length to the payload (`to_bytes`'s framing half), `frame_read_length` reads that header as an LE u32 (`read_length`), and `frame_decode` reproduces `from_bytes`'s two guards — `< 4` bytes → `data too short`, `< 4 + declared len` → `incomplete message` — then slices out `data[4 .. 4+len]`. Pure byte mechanics, so untyped; the bincode (de)serialization of the message *body* is the Rust-specific deferred piece (the caller (de)serializes the payload `frame_decode` hands back). LE u32 verified to round-trip `0x12345678` and `0xFFFFFFFF`. `make frame-check` pins the byte layout, the encode∘decode round-trip, both error guards, and the empty/zero-length-payload edge. | | `monitor::events::SuspiciousPatterns` (process-spawn classifier) | `jsecmon/suspicious.ss` | ✅ **untyped layer** — `check_suspicious(process, parent)`: shell-from-service, attack-tool-from-service (name exact-match or exe suffix), reverse-shell command-line patterns, and crypto-miner name/cmdline patterns, in secmon's order, returning the same reason string. Pure string classification like triage. Pins two corners the Rust depends on: a missing parent short-circuits to "clean" before any check, and `str::contains` is a *literal* substring test (so `python -c.*socket` is literal, not a regex). `make suspicious-check` reproduces secmon's two events.rs tests + the other three signals + both corners. | | `monitor::network::NetworkMonitor` (connection classifier) | `jsecmon/netconn.ss` | ✅ **untyped layer** — `check_suspicious(port, addr, process)`: known reverse-shell/C2/l33t port, ephemeral port (49152..65535) that is a round multiple of 1000, and a web-server process (nginx/apache/httpd/php-fpm) connecting to a non-private address, in secmon's order with the same reason string. Pure metadata classification. secmon hides the web-server names with `obfstr!` (same scheme as `typed/obfuscate.ss`); they decode to these plaintext literals at runtime. Pins the faithfulness quirk that the "private" prefix set is literal `{127. 10. 192.168. 172.}`, so `172.` matches all of 172.x, not just RFC1918 172.16/12. `make netconn-check` reproduces secmon's two network.rs tests + the full bad-port list + the high-port and web-server rules with private-address negatives. | new file mode 100644 --- /dev/null +++ b/examples/correlate_check.ss @@ -0,0 +1,114 @@ +;;; Parity check for (jsecmon correlate) against secmon storage/mod.rs detect_*. +;;; Reproduces test_detect_brute_force (+ below_threshold), _credential_stuffing, +;;; _dns_tunnel, _suspicious_cron and derives the bucket/threshold edges. The SQL +;;; fetch is the deferred boundary, so each test's stored events become the +;;; pre-shaped row list the detector consumes. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/correlate_check.ss + +(import (jerboa prelude) + (jsecmon correlate)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +(def (rule s) (cdr (assq 'rule s))) +(def (desc s) (cdr (assq 'description s))) +(def (sev s) (cdr (assq 'severity s))) +(def (ts s) (cdr (assq 'timestamp-ms s))) +(def (dval s k) (cdr (assq k (cdr (assq 'details s))))) + +;; ── brute force: 6 auth failures, user admin, bucket 1 (test_detect_…) ─────── +(displayln "detect-brute-force:") +(def bf-rows + (for/collect ((i (in-range 6))) (list "h1" "admin" (+ 600000 (* i 1000))))) +(def bf (detect-brute-force bf-rows)) +(check "6 failures -> 1 anomaly" (length bf) 1) +(check "rule" (rule (car bf)) "brute_force") +(check "severity" (sev (car bf)) "high") +(check "ts = bucket*600000" (ts (car bf)) 600000) +(check "failure-count detail" (dval (car bf) 'failure-count) 6) +(check "description" + (desc (car bf)) "6 auth failures for 'admin' on h1 in 10min") +;; below threshold: 4 failures -> nothing +(check "4 failures -> none" + (length (detect-brute-force + (for/collect ((i (in-range 4))) + (list "h1" "admin" (+ 600000 (* i 1000)))))) 0) +;; bucket boundary: 4 in bucket 1 + 1 in bucket 2 -> neither bucket reaches 5 +(check "split across buckets -> none" + (length (detect-brute-force + (list (list "h1" "admin" 600000) (list "h1" "admin" 601000) + (list "h1" "admin" 602000) (list "h1" "admin" 603000) + (list "h1" "admin" 1200000)))) 0) +;; two different users in the same bucket are separate groups +(check "distinct users -> separate groups" + (length (detect-brute-force + (append + (for/collect ((i (in-range 5))) (list "h1" "admin" (+ 600000 i))) + (for/collect ((i (in-range 5))) (list "h1" "root" (+ 600000 i)))))) 2) + +;; ── credential stuffing: 5 distinct users, same remote (test_detect_…) ─────── +(displayln "detect-credential-stuffing:") +(def cs-rows + (let ((users '("alice" "bob" "charlie" "dave" "eve"))) + (for/collect ((u users) (i (in-naturals))) + (list "h1" "10.0.0.99" u (+ 600000 (* i 1000)))))) +(def cs (detect-credential-stuffing cs-rows)) +(check "5 distinct users -> 1 anomaly" (length cs) 1) +(check "rule" (rule (car cs)) "credential_stuffing") +(check "distinct-usernames detail" (dval (car cs) 'distinct-usernames) 5) +(check "description" + (desc (car cs)) + "5 distinct users failing from 10.0.0.99 on h1 in 10min") +;; 5 rows but only 4 distinct users (alice repeats) -> below distinct threshold +(check "4 distinct (one repeat) -> none" + (length (detect-credential-stuffing + (list (list "h1" "10.0.0.99" "alice" 600000) + (list "h1" "10.0.0.99" "alice" 601000) + (list "h1" "10.0.0.99" "bob" 602000) + (list "h1" "10.0.0.99" "charlie" 603000) + (list "h1" "10.0.0.99" "dave" 604000)))) 0) + +;; ── DNS tunnel: 55 queries from dnscat, bucket 1 (test_detect_…) ───────────── +(displayln "detect-dns-tunnel:") +(def dt (detect-dns-tunnel + (for/collect ((i (in-range 55))) (list "h1" "dnscat" (+ 300000 (* i 100)))))) +(check "55 queries -> 1 anomaly" (length dt) 1) +(check "rule" (rule (car dt)) "dns_tunnel") +(check "ts = bucket*300000" (ts (car dt)) 300000) +(check "query-count detail" (dval (car dt) 'query-count) 55) +;; threshold is >=50: exactly 50 fires, 49 does not +(check "exactly 50 -> fires" + (length (detect-dns-tunnel + (for/collect ((i (in-range 50))) (list "h1" "p" (+ 300000 i))))) 1) +(check "49 -> none" + (length (detect-dns-tunnel + (for/collect ((i (in-range 49))) (list "h1" "p" (+ 300000 i))))) 0) + +;; ── suspicious cron: non-root user fires, root does not (test_detect_…) ────── +(displayln "detect-suspicious-cron:") +(def sc (detect-suspicious-cron + (list (list "h1" 1000 "modified /etc/crontab" "www-data") + (list "h1" 2000 "modified /etc/crontab" "root")))) +(check "1 non-root change -> 1 anomaly" (length sc) 1) +(check "rule" (rule (car sc)) "suspicious_cron") +(check "severity" (sev (car sc)) "medium") +(check "non-root user kept" (dval (car sc) 'user) "www-data") +(check "description" + (desc (car sc)) + "Cron change by non-root user 'www-data' on h1: modified /etc/crontab") +;; missing user -> "" (≠ root) -> kept, matching unwrap_or("") +(check "empty user kept" + (length (detect-suspicious-cron (list (list "h1" 5 "x" "")))) 1) +(check "all-root -> none" + (length (detect-suspicious-cron (list (list "h1" 1 "x" "root")))) 0) + +(newline) +(if (= fails 0) + (displayln "OK: correlate matches secmon's storage/mod.rs detect_* (batch 1).") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/correlate.ss @@ -0,0 +1,171 @@ +#!chezscheme +;;; jsecmon cross-event correlation rules (secmon storage/mod.rs detect_*), untyped. +;;; +;;; Each secmon detection rule is "run a SQL query, then turn the result rows +;;; into Anomaly records". The SQL fetch (and its json_extract / GROUP BY … +;;; HAVING) is the deferred storage I/O; what is pure and portable is the +;;; correlation *algorithm* over the fetched rows plus the Anomaly description. +;;; This module reimplements that algorithm directly over pre-shaped row lists so +;;; it can be checked without a database — which is exactly what storage/mod.rs's +;;; test_detect_* exercise (they insert events and assert the resulting Anomaly). +;;; +;;; Each detector returns a list of "anomaly seeds": an alist mirroring an +;;; Anomaly minus the calendar-formatted fields. The caller (storage layer) runs +;;; the timestamp fields named `*-ms` through format_ts for the final JSON and +;;; tags `attack` via (jsecmon detection-rules) anomaly-rule-attack. Seed shape: +;;; ((rule . s) (description . s) (severity . s) (timestamp-ms . n) +;;; (host . s) (details . <alist of pure fields + raw *-ms timestamps>)) +;;; +;;; Batch 1 — the GROUP BY … HAVING rules, reimplemented as pure tallies over +;;; rows the caller has already filtered to the rule's event subset: +;;; detect-brute-force rows=((host username ts) …) ≥5 / 10min +;;; detect-credential-stuffing rows=((host remote-host user ts) …) ≥5 distinct / 10min +;;; detect-dns-tunnel rows=((host proc-name ts) …) ≥50 / 5min +;;; detect-suspicious-cron rows=((host ts summary user) …) user≠"root" +;;; +;;; Faithfulness notes: +;;; * SQL `timestamp_ms / N` is integer division on a positive epoch → quotient; +;;; the bucket's anomaly timestamp is bucket*N (start of the 10-/5-min window). +;;; * GROUP BY result order is unspecified in SQL; seeds are emitted in +;;; first-seen group order (deterministic) — treat the set, not the order, as +;;; significant. COUNT(*) → tally-counts; COUNT(DISTINCT x) → tally-distinct. +;;; * brute_force/credential_stuffing feed only failed auth events +;;; (success=0); credential_stuffing's remote_host-IS-NOT-NULL and +;;; dns_tunnel's process_name-IS-NOT-NULL are likewise input preconditions +;;; (the caller's WHERE), so callers pass only qualifying rows. +;;; * suspicious_cron is a per-row filter (no windowing): keep rows whose +;;; parsed `user` field is not "root"; a missing user is "" (≠ root) → kept, +;;; matching unwrap_or(""). +;;; +;;; examples/correlate_check.ss reproduces storage/mod.rs's test_detect_brute_force +;;; (+ _below_threshold), test_detect_credential_stuffing, test_detect_dns_tunnel, +;;; test_detect_suspicious_cron and derives the bucket/threshold edges. + +(library (jsecmon correlate) + (export detect-brute-force detect-credential-stuffing + detect-dns-tunnel detect-suspicious-cron) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?)) + + ;; Canonical string key for a list of string/number parts (US-delimited). + (def (key->string parts) + (string-join (map (lambda (x) (str x)) parts) "\x1f;")) + + ;; COUNT(*) per key, returning ((key-parts . count) …) in first-seen order. + (def (tally-counts rows key-fn) + (let ((ht (make-hash-table)) (order '())) + (for-each + (lambda (row) + (let* ((parts (key-fn row)) (ks (key->string parts))) + (if (hash-key? ht ks) + (hash-put! ht ks (+ 1 (hash-ref ht ks))) + (begin (hash-put! ht ks 1) + (set! order (cons (cons ks parts) order)))))) + rows) + (map (lambda (p) (cons (cdr p) (hash-ref ht (car p)))) (reverse order)))) + + ;; COUNT(DISTINCT val) per key, returning ((key-parts . distinct-count) …). + (def (tally-distinct rows key-fn val-fn) + (let ((ht (make-hash-table)) (order '())) + (for-each + (lambda (row) + (let* ((parts (key-fn row)) (ks (key->string parts))) + (unless (hash-key? ht ks) + (hash-put! ht ks (make-hash-table)) + (set! order (cons (cons ks parts) order))) + (hash-put! (hash-ref ht ks) (str (val-fn row)) #t))) + rows) + (map (lambda (p) (cons (cdr p) (length (hash-keys (hash-ref ht (car p)))))) + (reverse order)))) + + ;; ── Rule 1: brute force — ≥5 auth failures, same user, 10-min bucket ──────── + (def (detect-brute-force rows) + (filter-map + (lambda (g) + (let* ((key (car g)) (count (cdr g)) + (host (car key)) (username (cadr key)) (bucket (caddr key))) + (and (>= count 5) + (let ((ts (* bucket 600000))) + (list (cons 'rule "brute_force") + (cons 'description + (str count " auth failures for '" username "' on " + host " in 10min")) + (cons 'severity "high") + (cons 'timestamp-ms ts) + (cons 'host host) + (cons 'details + (list (cons 'username username) + (cons 'failure-count count) + (cons 'window-start-ms ts)))))))) + (tally-counts rows + (lambda (r) (list (car r) (cadr r) (quotient (caddr r) 600000)))))) + + ;; ── Rule 2: credential stuffing — ≥5 distinct users, same remote, 10-min ──── + (def (detect-credential-stuffing rows) + (filter-map + (lambda (g) + (let* ((key (car g)) (ucount (cdr g)) + (host (car key)) (remote (cadr key)) (bucket (caddr key))) + (and (>= ucount 5) + (let ((ts (* bucket 600000))) + (list (cons 'rule "credential_stuffing") + (cons 'description + (str ucount " distinct users failing from " remote + " on " host " in 10min")) + (cons 'severity "high") + (cons 'timestamp-ms ts) + (cons 'host host) + (cons 'details + (list (cons 'remote-host remote) + (cons 'distinct-usernames ucount) + (cons 'window-start-ms ts)))))))) + (tally-distinct rows + (lambda (r) (list (car r) (cadr r) (quotient (cadddr r) 600000))) + (lambda (r) (caddr r))))) ;; distinct username + + ;; ── Rule 3: DNS tunnel — ≥50 queries, same process, 5-min bucket ──────────── + (def (detect-dns-tunnel rows) + (filter-map + (lambda (g) + (let* ((key (car g)) (count (cdr g)) + (host (car key)) (proc (cadr key)) (bucket (caddr key))) + (and (>= count 50) + (let ((ts (* bucket 300000))) + (list (cons 'rule "dns_tunnel") + (cons 'description + (str count " DNS queries from '" proc "' on " host + " in 5min")) + (cons 'severity "high") + (cons 'timestamp-ms ts) + (cons 'host host) + (cons 'details + (list (cons 'process-name proc) + (cons 'query-count count) + (cons 'window-start-ms ts)))))))) + (tally-counts rows + (lambda (r) (list (car r) (cadr r) (quotient (caddr r) 300000)))))) + + ;; ── Rule 4: suspicious cron — scheduled task change by non-root user ──────── + (def (detect-suspicious-cron rows) + (filter-map + (lambda (r) + (let ((host (car r)) (ts (cadr r)) (summary (caddr r)) (user (cadddr r))) + (and (not (string=? user "root")) + (list (cons 'rule "suspicious_cron") + (cons 'description + (str "Cron change by non-root user '" user "' on " + host ": " summary)) + (cons 'severity "medium") + (cons 'timestamp-ms ts) + (cons 'host host) + (cons 'details + (list (cons 'user user) (cons 'summary summary))))))) + rows)))