Port storage::entity_timeline field routing to pure Jerboa
ober
044e03a4f7b59021111abafc1fa93be86cc6b64c
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -75,6 +75,12 @@ storage-check: rust cd $(BUILD) && cargo build --release $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss +# storage::entity_timeline: the per-category WHERE-group builder (process/ip/ +# user/domain field routing) plus the assembled query over a live store. Pure +# storage layer, so no rust dep -- only the native sqlite loader. +entity-check: + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/entity_check.ss + # The SQL-aggregation threat detectors (brute_force, credential_stuffing, # dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) over a live store, # checked against secmon's run_detections test vectors. Needs the native lib too. @@ -323,6 +329,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/analytics_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/detect_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/storage_check.ss + $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/entity_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/threats_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_store_check.ss $(LOADER_ENV) SECMON_GEOIP_CSV="$(GEOIP_CSV)" $(SCHEME) --libdirs $(LIBDIRS) --script examples/geoip_check.ss --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `stealth::obfuscate` (compile-time string XOR) | `typed/obfuscate.ss` | ✅ **typed kernel** — keeps sensitive strings out of the binary in plaintext. Lowers secmon's `obfuscate!`/`obfuscate_bytes!` scheme: length-derived wrapping-u8 key (`len*31+42` / `len*37+13`) + XOR. Since XOR preserves length, decode recomputes the key from the buffer — a clean involutive pair, no stored key. `make test` reproduces secmon's `test_obfuscate_roundtrip` + `test_obfuscated_not_plaintext` plus key-derivation and all-256-byte round-trip vectors. | | `psk` HKDF/SHA256/AES-GCM | — | ⏳ FFI-delegated to vetted crates (not reimplemented) | | `crypto::ecies` | — | ⏳ FFI-delegated; orchestration only | -| `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). | +| `storage` (events table, store/query/filters) | `jsecmon/storage.ss` | ✅ **untyped layer** — SQLite event store on `(std db sqlite-native)` (rusqlite): secmon's schema (events + indexes + collector_state), `store-event` INSERT-OR-IGNORE dedup, and the full EventFilter WHERE builder (host/type/severity/since/until/pid/process_name LIKE/search/exclude_event_ids). `query-events` returns row hashes with `data` parsed from JSON, so detect/triage/analytics consume them directly. `make storage-check` round-trips store→query→detect→analytics (host risk 30, same as `detect-check`). `entity-where`/`entity-timeline` port `storage::entity_timeline`: the per-category WHERE-group builder (Process = `process_name LIKE` + six json fields with LIKE `%value%`; Ip/User/Domain = exact `json_extract` `=` over their field sets) plus the assembled oldest-first query, with the filter clauses continuing the positional `?N` numbering after the category binds (`build-where` gained an optional start index). `make entity-check` pins each category's exact SQL fragment + binds and routes/orders/narrows/limits over a live store (18 cases). | | `storage` SQL-aggregation detectors (brute_force, credential_stuffing, dns_tunnel, suspicious_cron, recon_port_scan, data_exfil) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `run_detections` family: the time-bucket GROUP BY/HAVING rules and the two 5-min sliding-window rules, run as SQL (json_extract) over a `(jsecmon storage)` handle. `make threats-check` reproduces secmon's six detection-rule test vectors. Remaining: the sequence/kill-chain rules (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell, impossible_travel) + frequency/severity/off-hours aggregates. | | `storage` sequence/chain detectors (priv_escalation_chain, persistence_after_access, log_cover, lateral_after_shell) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's `detect_sequence_pair` family: event A then event B within a window on the same host (auth-success→priv-esc /5min, reverse-shell/webshell→persistence /1h, any-critical→log-tampering /1h, shell→lateral /1h). Reproduces secmon's chain test vectors incl. the outside-window negative. | | `storage` time-window aggregates (frequency_spike, severity_cluster, off_hours, kill_chain) | `jsecmon/threats.ss` | ✅ **untyped layer** — secmon's full `detect_anomalies` family: per-(host,event_type) hour count 3x above its own average, 5+ crit/high on a host /5min, crit/high outside 08:00-18:00 UTC weekday (SQLite `strftime`), and 3+ distinct kill-chain phases /1h. `run-anomaly-detections` is the dispatcher (frequency_spike first, as secmon runs it). `make threats-check` covers each with threshold/negative cases. | new file mode 100644 --- /dev/null +++ b/examples/entity_check.ss @@ -0,0 +1,103 @@ +;;; Parity check for (jsecmon storage)'s entity-where / entity-timeline against +;;; secmon's storage::entity_timeline (src/storage/mod.rs). The pure part is the +;;; per-category WHERE-group builder (exact SQL fragment + positional binds); the +;;; end-to-end part proves the assembled query routes, orders ASC, narrows by the +;;; same EventFilter clauses, and honours an optional LIMIT over a real db. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/entity_check.ss + +(import (jerboa prelude) + (jsecmon storage)) + +(def fails 0) +(def (check label got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label + (if ok "" (str " got " got " want " want))))) + +;; ── pure entity-where: exact clause + binds per category ────────────────────── +(displayln "entity-where clauses:") +(def proc (entity-where 'process "curl")) +(check "process clause" + (car proc) + (str "process_name LIKE ?1" + " OR json_extract(data, '$.name') LIKE ?2" + " OR json_extract(data, '$.exe') LIKE ?3" + " OR json_extract(data, '$.source_process') LIKE ?4" + " OR json_extract(data, '$.spawned_process') LIKE ?5" + " OR json_extract(data, '$.web_server_name') LIKE ?6" + " OR json_extract(data, '$.parent_process') LIKE ?7")) +(check "process binds (7 x %curl%)" + (cdr proc) (list "%curl%" "%curl%" "%curl%" "%curl%" "%curl%" "%curl%" "%curl%")) + +(def ip (entity-where 'ip "10.0.0.5")) +(check "ip clause" + (car ip) + (str "json_extract(data, '$.remote_addr') = ?1" + " OR json_extract(data, '$.local_addr') = ?2" + " OR json_extract(data, '$.server_addr') = ?3" + " OR json_extract(data, '$.target_host') = ?4" + " OR json_extract(data, '$.remote_host') = ?5")) +(check "ip binds (5 x exact)" + (cdr ip) (list "10.0.0.5" "10.0.0.5" "10.0.0.5" "10.0.0.5" "10.0.0.5")) + +(def usr (entity-where 'user "alice")) +(check "user clause" + (car usr) + "json_extract(data, '$.username') = ?1 OR json_extract(data, '$.user') = ?2") +(check "user binds (2 x exact)" (cdr usr) (list "alice" "alice")) + +(def dom (entity-where 'domain "evil.com")) +(check "domain clause" (car dom) "json_extract(data, '$.query_name') = ?1") +(check "domain binds (1 x exact)" (cdr dom) (list "evil.com")) + +(check "unknown category errors" + (try (begin (entity-where 'bogus "x") 'no-error) + (catch (e) 'errored)) + 'errored) + +;; ── end-to-end entity-timeline over an in-memory store ──────────────────────── +(displayln "entity-timeline routing:") +(def (jdata . kvs) + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) (json-object->string h) + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + +(def db (store-open ":memory:")) +;; (db seq host source ts type severity pid pname summary data) +(store-event db 1 "h1" "agent" 1000 "process_start" "info" 100 "nginx" "" + (jdata "name" "nginx-worker" "exe" "/usr/sbin/nginx")) +(store-event db 2 "h1" "agent" 2000 "network_connect" "info" 200 "curl" "" + (jdata "exe" "/usr/bin/curl" "remote_addr" "10.0.0.5")) +(store-event db 3 "h1" "agent" 3000 "dns_query" "info" 300 "chrome" "" + (jdata "query_name" "evil.com")) +(store-event db 4 "h1" "agent" 1500 "auth_failure" "warn" 400 "sshd" "" + (jdata "username" "alice" "remote_addr" "10.0.0.5")) + +(def (ids rows) (map (lambda (r) (hash-get r "id")) rows)) +(def (tss rows) (map (lambda (r) (hash-get r "timestamp_ms")) rows)) + +;; process_name column + $.name + $.exe all funnel to the one nginx row +(check "process 'nginx' -> 1 row" (length (entity-timeline db 'process "nginx" (make-filter))) 1) +(check "process 'curl' -> e2" (ids (entity-timeline db 'process "curl" (make-filter))) (list 2)) +;; exact json match for domain / user +(check "domain 'evil.com' -> e3" (ids (entity-timeline db 'domain "evil.com" (make-filter))) (list 3)) +(check "domain miss -> none" (length (entity-timeline db 'domain "good.com" (make-filter))) 0) +(check "user 'alice' -> e4" (ids (entity-timeline db 'user "alice" (make-filter))) (list 4)) +;; ip touches two events (e4 @1500, e2 @2000); ORDER BY timestamp_ms ASC +(check "ip '10.0.0.5' -> 2 rows, oldest first" + (tss (entity-timeline db 'ip "10.0.0.5" (make-filter))) (list 1500 2000)) +;; filter clauses continue the bind numbering and narrow the group +(check "ip + since_ms 1600 -> e2 only" + (ids (entity-timeline db 'ip "10.0.0.5" (make-filter "since_ms" 1600))) (list 2)) +;; optional LIMIT (no default, unlike query-events) +(check "ip + limit 1 -> oldest only" + (ids (entity-timeline db 'ip "10.0.0.5" (make-filter "limit" 1))) (list 4)) +(store-close db) + +(newline) +(if (= fails 0) + (displayln "OK: entity-where/entity-timeline match secmon's storage::entity_timeline.") + (begin (displayln fails " FAILURES") (exit 1))) --- a/jsecmon/storage.ss +++ b/jsecmon/storage.ss @@ -21,7 +21,8 @@ (library (jsecmon storage) (export store-open store-close store-event store-count - make-filter query-events build-where) + make-filter query-events build-where + entity-where entity-timeline) (import (except (chezscheme) make-hash-table hash-table? sort sort! @@ -102,12 +103,20 @@ "SELECT id, seq, host, source, timestamp_ms, event_type, severity," " pid, process_name, summary, data FROM events WHERE 1=1")) + ;; Same columns, but the WHERE opens a group `(` the entity clause fills — + ;; entity_timeline wraps the category disjunction in parens, then appends the + ;; ordinary filter clauses (` AND …`) after it. + (def entity-select-prefix + (string-append + "SELECT id, seq, host, source, timestamp_ms, event_type, severity," + " pid, process_name, summary, data FROM events WHERE (")) + ;; Build the dynamic WHERE suffix + positional bind list, mirroring secmon's ;; append_filter_clauses. Returns (cons where-sql param-list). secmon reuses ;; one ?N for the two-column search; positional binding here is simplest with ;; two placeholders bound to the same value. - (def (build-where filter) - (let ((parts '()) (params '()) (idx 1)) + (def (build-where filter (start 1)) + (let ((parts '()) (params '()) (idx start)) (def (emit! frag val) (set! parts (cons frag parts)) (set! params (cons val params)) @@ -159,4 +168,57 @@ " ORDER BY timestamp_ms DESC" " LIMIT ?" idx " OFFSET ?" (+ idx 1))) (params (append where-params (list limit offset)))) + (map row->event (apply sqlite-query db sql params)))) + + ;; ── entity_timeline: per-category WHERE-group builder ──────────────────────── + ;; The pure routing core of secmon's storage::entity_timeline. Given a category + ;; (symbol process|ip|user|domain) and a value, returns (cons clause params): + ;; the SQL inside the WHERE `( … )` group plus its positional binds, numbered + ;; ?1.. in clause order. Process searches the process_name column plus six json + ;; fields with LIKE %value%; ip/user/domain match json fields exactly. + (def (entity-where category value) + (def (jx field op i) (str "json_extract(data, '" field "') " op " ?" i)) + ;; "f1 OR f2 OR …" over fields, numbering ?N from `start`, binding `val` + ;; (the same value) once per field. + (def (disjoin frag fields start val) + (let loop ((fs fields) (i start) (parts '()) (params '())) + (if (null? fs) + (cons (string-join (reverse parts) " OR ") (reverse params)) + (loop (cdr fs) (+ i 1) + (cons (frag (car fs) i) parts) + (cons val params))))) + (case category + ((process) + (let* ((like (str "%" value "%")) + (rest (disjoin (lambda (f i) (jx f "LIKE" i)) + '("$.name" "$.exe" "$.source_process" + "$.spawned_process" "$.web_server_name" "$.parent_process") + 2 like))) + (cons (str "process_name LIKE ?1 OR " (car rest)) (cons like (cdr rest))))) + ((ip) + (disjoin (lambda (f i) (jx f "=" i)) + '("$.remote_addr" "$.local_addr" "$.server_addr" "$.target_host" "$.remote_host") + 1 value)) + ((user) + (disjoin (lambda (f i) (jx f "=" i)) '("$.username" "$.user") 1 value)) + ((domain) + (disjoin (lambda (f i) (jx f "=" i)) '("$.query_name") 1 value)) + (else (error 'entity-where "unknown entity category" category)))) + + ;; Full entity_timeline query: events touching `value` in `category`, narrowed + ;; by the same EventFilter clauses, oldest-first. The category group binds first + ;; (?1..?k), the filter clauses continue the numbering (?k+1..), then an optional + ;; LIMIT — matching secmon (no default limit, no OFFSET). + (def (entity-timeline db category value filter) + (let* ((ew (entity-where category value)) + (eclause (car ew)) (eparams (cdr ew)) + (k (length eparams)) + (fw (build-where filter (+ k 1))) + (fclause (car fw)) (fparams (cdr fw)) + (limit (hash-get filter "limit")) + (lidx (+ k (length fparams) 1)) + (sql (str entity-select-prefix eclause ")" fclause + " ORDER BY timestamp_ms ASC" + (if limit (str " LIMIT ?" lidx) ""))) + (params (append eparams fparams (if limit (list limit) '())))) (map row->event (apply sqlite-query db sql params)))))