jsecmon: port storage IOC detection (new ioc.ss) + fix event-summary test note
ober
667c3e5d9d828d8e5793a70df98d6c469fce192f
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -241,6 +241,13 @@ collector-cli-check: event-summary-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss +# IOC type detection + text parsing (secmon src/storage/mod.rs): detect_ioc_type +# (ip/hash/domain/process first-match), is_ipv4 (4 short digit groups, no range +# check), parse_ioc_text (trim, drop blanks/# comments, tag). Mirrors the +# test_ioc_type_detection #[test]. +ioc-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/ioc_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -274,6 +281,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/analyze_cli_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_cli_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/ioc_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ make platform-mounts-check # is_dangerous_path (per-platform exact set) + get_mo make analyze-cli-check # analyze bin: parse_duration_ms + AlertSink::parse + --flag scanners make collector-cli-check # collector bin: --after/--format/--db + host normalize + hosts-file make event-summary-check # storage readers: extract_pid/extract_process_name/build_summary +make ioc-check # storage IOC: detect_ioc_type + is_ipv4 + parse_ioc_text make checks # every Jerboa-side check in one shot ``` @@ -132,5 +133,6 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `monitor::dns_sniffer` (DNS wire parser + dedup) | `jsecmon/dns-sniffer.ss` | ✅ **untyped layer** — the platform-independent half of secmon's `src/monitor/dns_sniffer.rs`: the DNS wire-format parser (QNAME decoding with compression-pointer chasing capped at 128 steps, QTYPE→string, question + A/AAAA answer-RR extraction) and the 5s dedup / 30s cleanup state machine. Every bounds check is preserved — a truncated/malformed/looping packet yields `#f`, never a bad read. Pure byte parsing → untyped, like geoip. The AF_PACKET raw-socket capture + `/proc` PID lookup stay for the monitor I/O driver. `make dns-sniffer-check` reproduces secmon's parser + dedup tests (+ AAAA, qtype table, pointer-loop/qdcount guards). | | `bin/analyze` (CLI parse helpers) | `jsecmon/analyze-cli.ss` | ✅ **untyped layer** — the pure argument parsers of the `analyze` binary, returning the prelude Result (ok/err) to mirror Rust's `Result<_, String>` **including the exact error text**: `parse_duration_ms` (`10m`/`2h`/`1d`/bare-seconds → ms; splits leading ASCII digits from the unit; empty → `empty duration`, bad number/leading-non-digit → `invalid duration: {s}`, bad unit like `m5` → `unknown duration unit: …`; the number must fit i64) and `AlertSink::parse` (`stdout` / `file:PATH` / `webhook:URL` / `syslog` / `syslog:TAG`, first-match in order, remainder taken verbatim so `file:` → empty path) and `parse_alert_sinks` (collect every `--alert-to <spec>`, parsing each and short-circuiting on the first bad spec like Rust's `?`; a trailing `--alert-to` with no value is skipped, and no flags → the empty list — the watch-time default-to-stdout lives in `cmd_watch`), plus the generic `--flag` scanners shared across the CLI (`parse_flag_value` → the arg after the **first** `flag`, or `#f` even when the flag is last; `has_flag` → membership; `is_json_format` → the first `--format` that has a value decides, a trailing `--format` is skipped). Pure string→Result/bool; the sink dispatch (stdout/file append/curl webhook/`logger` syslog) and query dispatch are the deferred I/O. secmon has no `#[test]` here so `make analyze-cli-check` asserts against the Rust source. (`format_ts`/`format_ts_iso` are chrono-calendar-coupled display helpers — deferred with the other calendar I/O.) | | `bin/collector` (CLI/hosts parse helpers) | `jsecmon/collector-cli.ss` | ✅ **untyped layer** — the pure argument/hosts parsing of the `collector` binary, with the async polling + ECIES/PSK key loading + SQLite I/O deferred: `parse_after_seq` (first `--after` value as u64, `unwrap_or(0)` so junk/negative/≥2⁶⁴ → 0), `parse_format` (→ `'json`/`'human`/`'quiet`; a per-index scan where an unknown `--format` value does **not** consume the value — differs from analyze's `is_json_format` — and the no-flag default is `quiet` when a `--db` is present else `human`), `parse_db_path`, `normalize_host` (append `:31337` unless the host already contains **any** `:`, so bare IPv6 is left as-is, faithfully), `collect_positional_hosts` (skip the four value flags **and** their values, drop other `--` args, normalize the rest), and `parse_hosts_file`'s pure contents→hosts core (trim, drop blanks/`#` comments, normalize). secmon has no `#[test]` here so `make collector-cli-check` asserts against the Rust source. | -| `storage` event readers (`extract_pid` / `extract_process_name` / `build_summary`) | `jsecmon/event-summary.ss` | ✅ **untyped layer** — the pure readers that turn an event's flat JSON `data` (a hash table, as `string->json-object` yields) back into a pid / process name / one-line summary, with the SQLite query + serde plumbing left to storage. Each field is read through a **typed** getter so only a JSON value of the right type counts (`as_u64`/`as_i64`/`as_str`/`as_bool`); `extract_pid` walks pid→source_pid→spawned_pid→web_server_pid and truncates the first hit to **u32** (Rust `v as u32`, so ≥2³² wraps, and pid 0 is a real hit); `extract_process_name` walks process_name→name→exe→source_process→spawned_process. `build_summary` reproduces every per-type format with the exact `unwrap_or` defaults (`"?"`/`0`), the `process_exit` exit-code *option* (Some(0) still prints `(0)`), the nested `selinux_event` perm/class/path-vs-message branches with the 80-char message cap, and the catch-all that scans values **in sorted key order** (serde's default BTreeMap) for the first string longer than 3 chars (capped at 80) else the event type. secmon has no `#[test]` here so `make event-summary-check` (43 cases) asserts against the Rust source. | +| `storage` event readers (`extract_pid` / `extract_process_name` / `build_summary`) | `jsecmon/event-summary.ss` | ✅ **untyped layer** — the pure readers that turn an event's flat JSON `data` (a hash table, as `string->json-object` yields) back into a pid / process name / one-line summary, with the SQLite query + serde plumbing left to storage. Each field is read through a **typed** getter so only a JSON value of the right type counts (`as_u64`/`as_i64`/`as_str`/`as_bool`); `extract_pid` walks pid→source_pid→spawned_pid→web_server_pid and truncates the first hit to **u32** (Rust `v as u32`, so ≥2³² wraps, and pid 0 is a real hit); `extract_process_name` walks process_name→name→exe→source_process→spawned_process. `build_summary` reproduces every per-type format with the exact `unwrap_or` defaults (`"?"`/`0`), the `process_exit` exit-code *option* (Some(0) still prints `(0)`), the nested `selinux_event` perm/class/path-vs-message branches with the 80-char message cap, and the catch-all that scans values **in sorted key order** (serde's default BTreeMap) for the first string longer than 3 chars (capped at 80) else the event type. `make event-summary-check` (43 cases) reproduces secmon's `test_extract_helpers` + `test_build_summary` and adds every per-type / typed-getter corner derived from the source. | +| `storage` IOC parsing (`detect_ioc_type` / `is_ipv4` / `parse_ioc_text`) | `jsecmon/ioc.ss` | ✅ **untyped layer** — the pure indicator classifier behind threat-list ingestion (the file read in `load_ioc_file` is the deferred I/O). `detect_ioc_type` is first-match ip→hash→domain→process: `is_ipv4` (split on `.`, exactly 4 non-empty ≤3-char all-digit groups — **no** 0–255 range check, so `999.999.999.999` is still Ip and `1.2.3.4444` is not), then IPv6 (`:` present and every char hex-or-`:`), then a 32/40/64-length all-hex Hash (MD5/SHA1/SHA256, case-insensitive), then a `.`-bearing space-free Domain, else Process. `parse_ioc_text` trims, drops blanks and `#` comments, and tags each remaining line (order preserved). `make ioc-check` reproduces the `test_ioc_type_detection` #[test] plus the is_ipv4 / parse corners. | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | new file mode 100644 --- /dev/null +++ b/examples/ioc_check.ss @@ -0,0 +1,70 @@ +;;; Parity check for (jsecmon ioc) against secmon src/storage/mod.rs. +;;; Reproduces the #[test] test_ioc_type_detection assertions verbatim and adds +;;; the is-ipv4 corners (no octet range check, empty segments) and parse-ioc-text. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/ioc_check.ss + +(import (jerboa prelude) + (jsecmon ioc)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +;; ── secmon test_ioc_type_detection (verbatim) ──────────────────────────────── +(displayln "test_ioc_type_detection:") +(check "192.168.1.1 -> ip" (detect-ioc-type "192.168.1.1") 'ip) +(check "10.0.0.1 -> ip" (detect-ioc-type "10.0.0.1") 'ip) +(check "fe80::1 -> ip" (detect-ioc-type "fe80::1") 'ip) +(check "MD5 -> hash" (detect-ioc-type "d41d8cd98f00b204e9800998ecf8427e") 'hash) +(check "SHA1 -> hash" (detect-ioc-type "da39a3ee5e6b4b0d3255bfef95601890afd80709") 'hash) +(check "SHA256 -> hash" + (detect-ioc-type "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") 'hash) +(check "evil.com -> domain" (detect-ioc-type "evil.com") 'domain) +(check "sub.evil.com -> domain" (detect-ioc-type "sub.evil.com") 'domain) +(check "nc -> process" (detect-ioc-type "nc") 'process) +(check "bash -> process" (detect-ioc-type "bash") 'process) + +;; ── detect-ioc-type derived corners ────────────────────────────────────────── +(displayln "detect-ioc-type corners:") +;; no 0-255 range check: 4 short digit groups are still Ip +(check "999.999.999.999 -> ip" (detect-ioc-type "999.999.999.999") 'ip) +;; a 4-digit octet fails is-ipv4; then no ':' , not hash len, has '.' no space -> domain +(check "1.2.3.4444 -> domain" (detect-ioc-type "1.2.3.4444") 'domain) +;; an upper-case hash still counts (ascii_hexdigit is case-insensitive) +(check "upper MD5 -> hash" + (detect-ioc-type "D41D8CD98F00B204E9800998ECF8427E") 'hash) +;; 31 hex chars is not a hash length -> no '.' -> process +(check "31 hex -> process" + (detect-ioc-type "d41d8cd98f00b204e9800998ecf8427") 'process) +;; ipv6 check requires hex-or-colon only: a non-hex letter falls through +(check "g::1 -> process" (detect-ioc-type "g::1") 'process) +;; '.' with a space is not a domain +(check "dotted with space -> process" (detect-ioc-type "a.b c") 'process) +(check "compact ipv6 -> ip" (detect-ioc-type "::1") 'ip) + +;; ── is-ipv4 directly ───────────────────────────────────────────────────────── +(displayln "is-ipv4:") +(check "1.2.3.4 is ipv4" (is-ipv4 "1.2.3.4") #t) +(check "3 parts not ipv4" (is-ipv4 "1.2.3") #f) +(check "5 parts not ipv4" (is-ipv4 "1.2.3.4.5") #f) +(check "trailing dot not ipv4" (is-ipv4 "1.2.3.") #f) +(check "leading dot not ipv4" (is-ipv4 ".1.2.3") #f) +(check "4-digit octet not ipv4" (is-ipv4 "1.2.3.4444") #f) +(check "non-digit not ipv4" (is-ipv4 "1.2.3.x") #f) + +;; ── parse-ioc-text: trim, drop blanks/# comments, tag each line ────────────── +(displayln "parse-ioc-text:") +(check "mixed list, order preserved" + (parse-ioc-text "192.168.1.1\n# a comment\n\n evil.com \nnc\n") + '(("192.168.1.1" ip) ("evil.com" domain) ("nc" process))) +(check "empty text -> empty" (parse-ioc-text "") '()) +(check "only comments -> empty" (parse-ioc-text "# x\n#y\n") '()) + +(newline) +(if (= fails 0) + (displayln "OK: ioc matches secmon's storage/mod.rs IOC detection.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/ioc.ss @@ -0,0 +1,82 @@ +#!chezscheme +;;; jsecmon IOC type detection + text parsing (secmon src/storage/mod.rs), untyped. +;;; +;;; The pure indicator-of-compromise helpers used to ingest threat lists, with +;;; the file read left to the storage layer (load_ioc_file's fs::read): +;;; is-ipv4 : string -> #t | #f +;;; detect-ioc-type : value -> 'ip | 'hash | 'domain | 'process +;;; parse-ioc-text : text -> ((value type) …) +;;; An Ioc is rendered here as a two-element list (value type-symbol); the four +;;; IocType variants map to the symbols 'ip / 'hash / 'domain / 'process. +;;; +;;; Faithful corners (mirroring detect_ioc_type's first-match order): +;;; 1. is-ipv4: split on '.', EXACTLY 4 parts, each non-empty, ≤ 3 chars, all +;;; ASCII digits — NO 0–255 range check, so "999.999.999.999" is still Ip +;;; and a 4-digit octet ("1.2.3.4444") fails. Leading/trailing dots keep an +;;; empty segment (Rust split), so ".1.2.3" / "1.2.3." are not IPv4. +;;; 2. else IPv6: contains ':' AND every char is a hex digit or ':'. +;;; 3. else Hash: length is exactly 32 / 40 / 64 (MD5 / SHA1 / SHA256) AND all +;;; ASCII hex digits. +;;; 4. else Domain: contains '.' AND contains no space. +;;; 5. else Process. +;;; parse-ioc-text trims each line, drops blanks and '#' comments, and tags each +;;; remaining line with detect-ioc-type — order preserved. +;;; +;;; Mirrors secmon's #[test] test_ioc_type_detection (examples/ioc_check.ss +;;; reproduces every assertion and adds the is-ipv4 / parse-ioc-text corners). + +(library (jsecmon ioc) + (export is-ipv4 detect-ioc-type parse-ioc-text) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?)) + + (def (ascii-digit? c) (and (char>=? c #\0) (char<=? c #\9))) + (def (ascii-hexdigit? c) + (or (ascii-digit? c) + (and (char>=? c #\a) (char<=? c #\f)) + (and (char>=? c #\A) (char<=? c #\F)))) + + ;; #t iff every char of s satisfies pred (vacuously #t for ""). + (def (string-all? pred s) + (let ((n (string-length s))) + (let loop ((i 0)) + (or (>= i n) (and (pred (string-ref s i)) (loop (+ i 1))))))) + + (def (is-ipv4 s) + (let ((parts (string-split s #\.))) + (and (= (length parts) 4) + (every (lambda (p) + (and (not (string-empty? p)) + (<= (string-length p) 3) + (string-all? ascii-digit? p))) + parts)))) + + (def (detect-ioc-type value) + (cond + ((is-ipv4 value) 'ip) + ((and (string-contains value ":") + (string-all? (lambda (c) (or (ascii-hexdigit? c) (char=? c #\:))) value)) + 'ip) + ((and (memv (string-length value) '(32 40 64)) + (string-all? ascii-hexdigit? value)) + 'hash) + ((and (string-contains value ".") (not (string-contains value " "))) + 'domain) + (#t 'process))) + + (def (parse-ioc-text text) + (filter-map + (lambda (l) + (let ((t (string-trim l))) + (and (not (string-empty? t)) + (not (string-prefix? "#" t)) + (list t (detect-ioc-type t))))) + (string-split text #\newline))))