jsecmon: port storage event readers (new event-summary.ss)
ober
76f61c93aa3ab96cc318aae0a4cba2236648250b
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -235,6 +235,12 @@ analyze-cli-check: collector-cli-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_cli_check.ss +# event summary/extract readers (secmon src/storage/mod.rs): extract_pid (u32 +# truncation), extract_process_name (field-priority), build_summary (per-type +# one-liners with the unwrap_or defaults + the sorted-key string fallback). +event-summary-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -267,6 +273,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/platform_mounts_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/analyze_cli_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_cli_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ make webshell-check # web-server-spawned suspicious child: name/cmdline classif make platform-mounts-check # is_dangerous_path (per-platform exact set) + get_mounts line parsers make analyze-cli-check # analyze bin: parse_duration_ms + AlertSink::parse + --flag scanners make collector-cli-check # collector bin: --after/--format/--db + host normalize + hosts-file +make event-summary-check # storage readers: extract_pid/extract_process_name/build_summary make checks # every Jerboa-side check in one shot ``` @@ -131,4 +132,5 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `monitor::dns_sniffer` (DNS wire parser + dedup) | `jsecmon/dns-sniffer.ss` | ✅ **untyped layer** — the platform-independent half of secmon's `src/monitor/dns_sniffer.rs`: the DNS wire-format parser (QNAME decoding with compression-pointer chasing capped at 128 steps, QTYPE→string, question + A/AAAA answer-RR extraction) and the 5s dedup / 30s cleanup state machine. Every bounds check is preserved — a truncated/malformed/looping packet yields `#f`, never a bad read. Pure byte parsing → untyped, like geoip. The AF_PACKET raw-socket capture + `/proc` PID lookup stay for the monitor I/O driver. `make dns-sniffer-check` reproduces secmon's parser + dedup tests (+ AAAA, qtype table, pointer-loop/qdcount guards). | | `bin/analyze` (CLI parse helpers) | `jsecmon/analyze-cli.ss` | ✅ **untyped layer** — the pure argument parsers of the `analyze` binary, returning the prelude Result (ok/err) to mirror Rust's `Result<_, String>` **including the exact error text**: `parse_duration_ms` (`10m`/`2h`/`1d`/bare-seconds → ms; splits leading ASCII digits from the unit; empty → `empty duration`, bad number/leading-non-digit → `invalid duration: {s}`, bad unit like `m5` → `unknown duration unit: …`; the number must fit i64) and `AlertSink::parse` (`stdout` / `file:PATH` / `webhook:URL` / `syslog` / `syslog:TAG`, first-match in order, remainder taken verbatim so `file:` → empty path) and `parse_alert_sinks` (collect every `--alert-to <spec>`, parsing each and short-circuiting on the first bad spec like Rust's `?`; a trailing `--alert-to` with no value is skipped, and no flags → the empty list — the watch-time default-to-stdout lives in `cmd_watch`), plus the generic `--flag` scanners shared across the CLI (`parse_flag_value` → the arg after the **first** `flag`, or `#f` even when the flag is last; `has_flag` → membership; `is_json_format` → the first `--format` that has a value decides, a trailing `--format` is skipped). Pure string→Result/bool; the sink dispatch (stdout/file append/curl webhook/`logger` syslog) and query dispatch are the deferred I/O. secmon has no `#[test]` here so `make analyze-cli-check` asserts against the Rust source. (`format_ts`/`format_ts_iso` are chrono-calendar-coupled display helpers — deferred with the other calendar I/O.) | | `bin/collector` (CLI/hosts parse helpers) | `jsecmon/collector-cli.ss` | ✅ **untyped layer** — the pure argument/hosts parsing of the `collector` binary, with the async polling + ECIES/PSK key loading + SQLite I/O deferred: `parse_after_seq` (first `--after` value as u64, `unwrap_or(0)` so junk/negative/≥2⁶⁴ → 0), `parse_format` (→ `'json`/`'human`/`'quiet`; a per-index scan where an unknown `--format` value does **not** consume the value — differs from analyze's `is_json_format` — and the no-flag default is `quiet` when a `--db` is present else `human`), `parse_db_path`, `normalize_host` (append `:31337` unless the host already contains **any** `:`, so bare IPv6 is left as-is, faithfully), `collect_positional_hosts` (skip the four value flags **and** their values, drop other `--` args, normalize the rest), and `parse_hosts_file`'s pure contents→hosts core (trim, drop blanks/`#` comments, normalize). secmon has no `#[test]` here so `make collector-cli-check` asserts against the Rust source. | +| `storage` event readers (`extract_pid` / `extract_process_name` / `build_summary`) | `jsecmon/event-summary.ss` | ✅ **untyped layer** — the pure readers that turn an event's flat JSON `data` (a hash table, as `string->json-object` yields) back into a pid / process name / one-line summary, with the SQLite query + serde plumbing left to storage. Each field is read through a **typed** getter so only a JSON value of the right type counts (`as_u64`/`as_i64`/`as_str`/`as_bool`); `extract_pid` walks pid→source_pid→spawned_pid→web_server_pid and truncates the first hit to **u32** (Rust `v as u32`, so ≥2³² wraps, and pid 0 is a real hit); `extract_process_name` walks process_name→name→exe→source_process→spawned_process. `build_summary` reproduces every per-type format with the exact `unwrap_or` defaults (`"?"`/`0`), the `process_exit` exit-code *option* (Some(0) still prints `(0)`), the nested `selinux_event` perm/class/path-vs-message branches with the 80-char message cap, and the catch-all that scans values **in sorted key order** (serde's default BTreeMap) for the first string longer than 3 chars (capped at 80) else the event type. secmon has no `#[test]` here so `make event-summary-check` (43 cases) asserts against the Rust source. | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | new file mode 100644 --- /dev/null +++ b/examples/event_summary_check.ss @@ -0,0 +1,146 @@ +;;; Parity check for (jsecmon event-summary) against secmon src/storage/mod.rs +;;; (extract_pid, extract_process_name, build_summary). secmon has no #[test] +;;; here, so this derives expectations from the Rust source and IS the spec for +;;; the port. `data` objects are built with the real JSON parser. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/event_summary_check.ss + +(import (jerboa prelude) + (jsecmon event-summary)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +(def (j s) (string->json-object s)) ;; flat JSON data object (a hash table) + +;; ── extract-pid: first present u64 field, truncated to u32 ─────────────────── +(displayln "extract-pid:") +(check "pid" (extract-pid (j "{\"pid\": 1234}")) 1234) +(check "source_pid" (extract-pid (j "{\"source_pid\": 99}")) 99) +(check "spawned_pid" (extract-pid (j "{\"spawned_pid\": 7}")) 7) +(check "web_server_pid" (extract-pid (j "{\"web_server_pid\": 5}")) 5) +(check "none -> #f" (extract-pid (j "{\"foo\": 1}")) #f) +(check "pid 0 is a hit" (extract-pid (j "{\"pid\": 0}")) 0) +;; `v as u32` truncates: 2^32 wraps to 0 +(check "2^32 truncates to 0" (extract-pid (j "{\"pid\": 4294967296}")) 0) +;; a non-u64 pid (string / negative) is skipped, next field tried +(check "string pid skipped" (extract-pid (j "{\"pid\": \"x\", \"source_pid\": 3}")) 3) +(check "negative pid skipped" (extract-pid (j "{\"pid\": -1, \"source_pid\": 8}")) 8) + +;; ── extract-process-name: first present string field ───────────────────────── +(displayln "extract-process-name:") +(check "process_name" (extract-process-name (j "{\"process_name\": \"sshd\"}")) "sshd") +(check "name" (extract-process-name (j "{\"name\": \"bash\"}")) "bash") +(check "exe" (extract-process-name (j "{\"exe\": \"/bin/x\"}")) "/bin/x") +(check "source_process" (extract-process-name (j "{\"source_process\": \"a\"}")) "a") +(check "spawned_process" (extract-process-name (j "{\"spawned_process\": \"b\"}")) "b") +(check "none -> #f" (extract-process-name (j "{\"pid\": 1}")) #f) +(check "process_name wins" + (extract-process-name (j "{\"process_name\": \"p\", \"name\": \"n\"}")) "p") + +;; ── build-summary: per-type formats with unwrap_or defaults ────────────────── +(displayln "build-summary:") +(check "process_start" + (build-summary "process_start" (j "{\"name\":\"bash\",\"pid\":42,\"exe\":\"/bin/bash\"}")) + "bash (pid 42) exec /bin/bash") +(check "process_start defaults" + (build-summary "process_start" (j "{}")) "? (pid 0) exec ?") +;; exit_code is an i64 OPTION: Some(0) still prints parens; absent prints none +(check "process_exit with code" + (build-summary "process_exit" (j "{\"pid\":10,\"exit_code\":0}")) "pid 10 exited (0)") +(check "process_exit no code" + (build-summary "process_exit" (j "{\"pid\":10}")) "pid 10 exited") +(check "process_exit negative code" + (build-summary "process_exit" (j "{\"pid\":5,\"exit_code\":-9}")) "pid 5 exited (-9)") +(check "dns_query" + (build-summary "dns_query" (j "{\"query_name\":\"evil.com\",\"query_type\":\"A\"}")) + "evil.com A query") +(check "network_connection" + (build-summary "network_connection" + (j "{\"process_name\":\"curl\",\"remote_addr\":\"1.2.3.4\",\"remote_port\":443}")) + "curl -> 1.2.3.4:443") +(check "suspicious_exec -> reason" + (build-summary "suspicious_exec" (j "{\"reason\":\"r\"}")) "r") +(check "suspicious_file_change -> reason" + (build-summary "suspicious_file_change" (j "{\"reason\":\"changed\"}")) "changed") +(check "reverse_shell" + (build-summary "reverse_shell" + (j "{\"process_name\":\"sh\",\"remote_addr\":\"10.0.0.1\",\"remote_port\":4444}")) + "sh -> 10.0.0.1:4444") +(check "persistence_event" + (build-summary "persistence_event" (j "{\"mechanism\":\"cron\",\"path\":\"/etc/cron.d/x\"}")) + "cron: /etc/cron.d/x") +(check "lateral_movement" + (build-summary "lateral_movement" + (j "{\"movement_type\":\"ssh\",\"target_host\":\"h2\",\"target_port\":22}")) + "ssh -> h2:22") +(check "log_tampering" + (build-summary "log_tampering" + (j "{\"tamper_type\":\"truncate\",\"log_path\":\"/var/log/auth.log\"}")) + "truncate: /var/log/auth.log") +(check "webshell" + (build-summary "webshell" + (j "{\"web_server_name\":\"nginx\",\"spawned_process\":\"sh\"}")) + "nginx spawned sh") +(check "auth_event ok" + (build-summary "auth_event" + (j "{\"auth_type\":\"ssh\",\"username\":\"root\",\"success\":true}")) + "ssh root ok") +(check "auth_event FAILED" + (build-summary "auth_event" + (j "{\"auth_type\":\"ssh\",\"username\":\"root\",\"success\":false}")) + "ssh root FAILED") +(check "auth_event missing success -> FAILED" + (build-summary "auth_event" (j "{\"auth_type\":\"ssh\",\"username\":\"root\"}")) + "ssh root FAILED") +(check "selinux full (perm+class+path)" + (build-summary "selinux_event" + (j "{\"event_type\":\"avc\",\"process_name\":\"httpd\",\"permission\":\"read\",\"target_class\":\"file\",\"path\":\"/etc/shadow\"}")) + "avc httpd denied read file on /etc/shadow") +(check "selinux no path" + (build-summary "selinux_event" + (j "{\"event_type\":\"avc\",\"process_name\":\"httpd\",\"permission\":\"read\",\"target_class\":\"file\"}")) + "avc httpd denied read file") +(check "selinux message branch" + (build-summary "selinux_event" + (j "{\"event_type\":\"avc\",\"process_name\":\"httpd\",\"message\":\"some denial text\"}")) + "avc httpd: some denial text") +(check "selinux bare" + (build-summary "selinux_event" + (j "{\"event_type\":\"avc\",\"process_name\":\"httpd\"}")) + "avc httpd") +(check "privilege_escalation" + (build-summary "privilege_escalation" + (j "{\"old_uid\":1000,\"new_uid\":0,\"method\":\"sudo\"}")) + "uid 1000 -> 0 via sudo") +(check "heartbeat" + (build-summary "heartbeat" (j "{\"uptime_secs\":3600}")) "uptime 3600s") +(check "agent_start" + (build-summary "agent_start" (j "{\"agent_hostname\":\"node1\"}")) + "agent started on node1") + +;; ── build-summary catch-all: first string (len > 3) in SORTED key order ────── +(displayln "build-summary fallback:") +;; sorted keys alpha<zeta; alpha's value too short -> zeta's value chosen +(check "sorted order, skip short" + (build-summary "mystery" (j "{\"alpha\":\"ab\",\"zeta\":\"valuehere\"}")) + "valuehere") +;; len exactly 3 is NOT > 3, so it is skipped -> falls through to event type +(check "len 3 skipped -> event type" + (build-summary "mystery" (j "{\"a\":\"abc\"}")) "mystery") +(check "no usable string -> event type" + (build-summary "weird" (j "{\"n\": 5}")) "weird") +;; the chosen string is capped at 80 chars +(def big (make-hash-table)) +(hash-put! big "blob" (make-string 90 #\x)) +(check "fallback caps at 80 chars" + (build-summary "weird" big) (make-string 80 #\x)) + +(newline) +(if (= fails 0) + (displayln "OK: event-summary matches secmon's storage/mod.rs readers.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/event-summary.ss @@ -0,0 +1,159 @@ +#!chezscheme +;;; jsecmon event summary/extract helpers (secmon src/storage/mod.rs), untyped. +;;; +;;; The pure readers over an event's flat JSON `data` object — the side that +;;; turns a stored event row back into a pid / process name / one-line summary, +;;; with the SQLite query + serde plumbing left to the storage layer: +;;; extract-pid : data -> u32 | #f +;;; extract-process-name : data -> string | #f +;;; build-summary : event-type data -> string +;;; `data` is the parsed flat JSON object — a hash table, exactly what +;;; (string->json-object …) yields (JSON numbers come back as exact integers, +;;; true/false as #t/#f, a missing key as #f). +;;; +;;; Faithful corners (mirroring serde_json's typed accessors): +;;; * a field is only used when its JSON type matches: `as_u64` wants a +;;; non-negative integer < 2^64, `as_i64` a signed-64 integer, `as_str` a +;;; string, `as_bool` a boolean — anything else reads as absent. We model +;;; each with a typed getter so a string "42" never counts as a number, etc. +;;; * extract-pid walks pid / source_pid / spawned_pid / web_server_pid and +;;; returns the first present u64 **truncated to u32** (Rust `v as u32`), so +;;; a value ≥ 2^32 wraps — and pid 0 is a real hit (0 is truthy here). +;;; * extract-process-name walks process_name / name / exe / source_process / +;;; spawned_process, first string wins. +;;; * build-summary's per-type formats use the Rust `unwrap_or` defaults +;;; verbatim ("?" for strings, 0 for numbers); process_exit's exit_code is an +;;; i64 *option* (Some -> " (c)", None -> no parens). selinux_event keeps the +;;; nested perm/class/path-vs-message branching and the 80-char message cap. +;;; The catch-all scans the object's values **in sorted key order** (serde's +;;; default Map is a BTreeMap — no preserve_order feature) and returns the +;;; first string longer than 3 chars (capped at 80), else the event type. +;;; +;;; secmon has no #[test] for these, so examples/event_summary_check.ss derives +;;; every expectation from the Rust source and IS the spec for this port. + +(library (jsecmon event-summary) + (export extract-pid extract-process-name build-summary) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?)) + + ;; typed JSON-field getters: the value at `key` only if it has the wanted + ;; type, else #f (so a missing key, a null, or a type mismatch all read #f). + (def (js-str data key) + (let ((v (hash-get data key))) (and (string? v) v))) + (def (js-str-or data key dflt) + (or (js-str data key) dflt)) + (def (js-u64 data key) + (let ((v (hash-get data key))) + (and (integer? v) (>= v 0) (< v (expt 2 64)) v))) + (def (js-u64-or data key dflt) + (or (js-u64 data key) dflt)) + (def (js-i64 data key) + (let ((v (hash-get data key))) + (and (integer? v) (>= v (- (expt 2 63))) (< v (expt 2 63)) v))) + (def (js-true? data key) + (eq? (hash-get data key) #t)) + + (def (first-n-chars s n) + (substring s 0 (min n (string-length s)))) + + ;; first present u64 field, truncated to u32 like Rust's `v as u32`. + (def (u32-of data key) + (let ((v (js-u64 data key))) + (and v (bitwise-and v #xffffffff)))) + + (def (extract-pid data) + (or (u32-of data "pid") + (u32-of data "source_pid") + (u32-of data "spawned_pid") + (u32-of data "web_server_pid"))) + + (def (extract-process-name data) + (or (js-str data "process_name") + (js-str data "name") + (js-str data "exe") + (js-str data "source_process") + (js-str data "spawned_process"))) + + ;; the catch-all: first string value (len > 3) in sorted key order, capped at + ;; 80 chars; else the event-type string itself. + (def (summary-fallback event-type data) + (let loop ((ks (list-sort string<? (hash-keys data)))) + (if (null? ks) + event-type + (let ((v (hash-get data (car ks)))) + (if (and (string? v) (> (string-length v) 3)) + (first-n-chars v 80) + (loop (cdr ks))))))) + + (def (build-summary event-type data) + (cond + ((string=? event-type "process_start") + (str (js-str-or data "name" "?") " (pid " (js-u64-or data "pid" 0) + ") exec " (js-str-or data "exe" "?"))) + ((string=? event-type "process_exit") + (let ((pid (js-u64-or data "pid" 0)) + (code (js-i64 data "exit_code"))) + (if code + (str "pid " pid " exited (" code ")") + (str "pid " pid " exited")))) + ((string=? event-type "dns_query") + (str (js-str-or data "query_name" "?") " " + (js-str-or data "query_type" "?") " query")) + ((string=? event-type "network_connection") + (str (js-str-or data "process_name" "?") " -> " + (js-str-or data "remote_addr" "?") ":" + (js-u64-or data "remote_port" 0))) + ((or (string=? event-type "suspicious_exec") + (string=? event-type "suspicious_connection") + (string=? event-type "suspicious_file_change")) + (js-str-or data "reason" "?")) + ((string=? event-type "reverse_shell") + (str (js-str-or data "process_name" "?") " -> " + (js-str-or data "remote_addr" "?") ":" + (js-u64-or data "remote_port" 0))) + ((string=? event-type "persistence_event") + (str (js-str-or data "mechanism" "?") ": " (js-str-or data "path" "?"))) + ((string=? event-type "lateral_movement") + (str (js-str-or data "movement_type" "?") " -> " + (js-str-or data "target_host" "?") ":" + (js-u64-or data "target_port" 0))) + ((string=? event-type "log_tampering") + (str (js-str-or data "tamper_type" "?") ": " + (js-str-or data "log_path" "?"))) + ((string=? event-type "webshell") + (str (js-str-or data "web_server_name" "?") " spawned " + (js-str-or data "spawned_process" "?"))) + ((string=? event-type "auth_event") + (str (js-str-or data "auth_type" "?") " " (js-str-or data "username" "?") + " " (if (js-true? data "success") "ok" "FAILED"))) + ((string=? event-type "selinux_event") + (let ((etype (js-str-or data "event_type" "?")) + (proc (js-str-or data "process_name" "?")) + (perm (js-str-or data "permission" "")) + (class (js-str-or data "target_class" "")) + (path (js-str-or data "path" ""))) + (if (and (not (string=? perm "")) (not (string=? class ""))) + (if (not (string=? path "")) + (str etype " " proc " denied " perm " " class " on " path) + (str etype " " proc " denied " perm " " class)) + (let ((msg (js-str-or data "message" ""))) + (if (not (string=? msg "")) + (str etype " " proc ": " (first-n-chars msg 80)) + (str etype " " proc)))))) + ((string=? event-type "privilege_escalation") + (str "uid " (js-u64-or data "old_uid" 0) " -> " + (js-u64-or data "new_uid" 0) " via " (js-str-or data "method" "?"))) + ((string=? event-type "heartbeat") + (str "uptime " (js-u64-or data "uptime_secs" 0) "s")) + ((string=? event-type "agent_start") + (str "agent started on " (js-str-or data "agent_hostname" "?"))) + (#t (summary-fallback event-type data)))))