jsecmon: port collector.rs CLI/hosts parsers (new collector-cli.ss)
ober
ac1f1b9198427998d3fb72e62e35d3ed1a7e28fd
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -229,6 +229,12 @@ platform-mounts-check: analyze-cli-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/analyze_cli_check.ss +# collector CLI parse helpers (secmon src/bin/collector.rs): parse_after_seq, +# parse_format (json/human/quiet), parse_db_path, normalize_host (default port +# 31337), collect_positional_hosts, and parse_hosts_file's contents->hosts core. +collector-cli-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_cli_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -260,6 +266,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/webshell_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/platform_mounts_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/analyze_cli_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_cli_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ make file-change-check # is_suspicious_change: setuid/setgid added, critical fil make webshell-check # web-server-spawned suspicious child: name/cmdline classifier + reason make platform-mounts-check # is_dangerous_path (per-platform exact set) + get_mounts line parsers make analyze-cli-check # analyze bin: parse_duration_ms + AlertSink::parse + --flag scanners +make collector-cli-check # collector bin: --after/--format/--db + host normalize + hosts-file make checks # every Jerboa-side check in one shot ``` @@ -129,4 +130,5 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `monitor/webshell` (`WebshellMonitor` classifiers) | `jsecmon/webshell.ss` | ✅ **untyped layer** — the three pure deciders with the `/proc` scan + parent/child PID walk + event emission stripped: `is_web_server` (lower-cased name **substring** vs the server list, so `php-fpm` matches `php`), `is_suspicious_child` (process name by **exact** lower-cased equality — `bashx` is clean — OR the joined+lowercased cmdline **substring**-matched against the pattern list), and `get_detection_reason` (scans only the cmdline patterns, **in list order**, first match → `Suspicious command pattern: {pat}`, else the default `Web server spawned suspicious process: {name}` with the **original-case** name). obfstr!-hidden lists decode to these plaintext literals. Pins the corner that the reason is chosen by pattern-list order, not cmdline-token order, and that a name-only hit yields the default reason. Pure — the PID walk is the deferred monitor loop — no native lib; secmon has no `#[test]` here so `make webshell-check` asserts against the Rust source. | | `monitor::dns_sniffer` (DNS wire parser + dedup) | `jsecmon/dns-sniffer.ss` | ✅ **untyped layer** — the platform-independent half of secmon's `src/monitor/dns_sniffer.rs`: the DNS wire-format parser (QNAME decoding with compression-pointer chasing capped at 128 steps, QTYPE→string, question + A/AAAA answer-RR extraction) and the 5s dedup / 30s cleanup state machine. Every bounds check is preserved — a truncated/malformed/looping packet yields `#f`, never a bad read. Pure byte parsing → untyped, like geoip. The AF_PACKET raw-socket capture + `/proc` PID lookup stay for the monitor I/O driver. `make dns-sniffer-check` reproduces secmon's parser + dedup tests (+ AAAA, qtype table, pointer-loop/qdcount guards). | | `bin/analyze` (CLI parse helpers) | `jsecmon/analyze-cli.ss` | ✅ **untyped layer** — the pure argument parsers of the `analyze` binary, returning the prelude Result (ok/err) to mirror Rust's `Result<_, String>` **including the exact error text**: `parse_duration_ms` (`10m`/`2h`/`1d`/bare-seconds → ms; splits leading ASCII digits from the unit; empty → `empty duration`, bad number/leading-non-digit → `invalid duration: {s}`, bad unit like `m5` → `unknown duration unit: …`; the number must fit i64) and `AlertSink::parse` (`stdout` / `file:PATH` / `webhook:URL` / `syslog` / `syslog:TAG`, first-match in order, remainder taken verbatim so `file:` → empty path), plus the generic `--flag` scanners shared across the CLI (`parse_flag_value` → the arg after the **first** `flag`, or `#f` even when the flag is last; `has_flag` → membership; `is_json_format` → the first `--format` that has a value decides, a trailing `--format` is skipped). Pure string→Result/bool; the sink dispatch (stdout/file append/curl webhook/`logger` syslog) and query dispatch are the deferred I/O. secmon has no `#[test]` here so `make analyze-cli-check` asserts against the Rust source. (`format_ts`/`format_ts_iso` are chrono-calendar-coupled display helpers — deferred with the other calendar I/O.) | +| `bin/collector` (CLI/hosts parse helpers) | `jsecmon/collector-cli.ss` | ✅ **untyped layer** — the pure argument/hosts parsing of the `collector` binary, with the async polling + ECIES/PSK key loading + SQLite I/O deferred: `parse_after_seq` (first `--after` value as u64, `unwrap_or(0)` so junk/negative/≥2⁶⁴ → 0), `parse_format` (→ `'json`/`'human`/`'quiet`; a per-index scan where an unknown `--format` value does **not** consume the value — differs from analyze's `is_json_format` — and the no-flag default is `quiet` when a `--db` is present else `human`), `parse_db_path`, `normalize_host` (append `:31337` unless the host already contains **any** `:`, so bare IPv6 is left as-is, faithfully), `collect_positional_hosts` (skip the four value flags **and** their values, drop other `--` args, normalize the rest), and `parse_hosts_file`'s pure contents→hosts core (trim, drop blanks/`#` comments, normalize). secmon has no `#[test]` here so `make collector-cli-check` asserts against the Rust source. | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | new file mode 100644 --- /dev/null +++ b/examples/collector_cli_check.ss @@ -0,0 +1,99 @@ +;;; Parity check for (jsecmon collector-cli) against secmon src/bin/collector.rs +;;; (parse_after_seq, parse_format, parse_db_path, normalize_host, +;;; collect_positional_hosts, parse_hosts_file's contents->hosts core). secmon +;;; has no #[test] here, so this derives expectations from the Rust source and +;;; IS the spec for the port. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/collector_cli_check.ss + +(import (jerboa prelude) + (jsecmon collector-cli)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +;; ── parse-after-seq: first --after with a value -> u64, else 0 ─────────────── +(displayln "parse-after-seq:") +(check "after value" (parse-after-seq '("--after" "1000")) 1000) +(check "after mid-list" (parse-after-seq '("watch" "--after" "42" "--db" "x")) 42) +(check "no --after -> 0" (parse-after-seq '("watch" "--db" "x")) 0) +(check "trailing --after -> 0" (parse-after-seq '("watch" "--after")) 0) +;; .parse::<u64>().unwrap_or(0): non-numeric / negative / >= 2^64 -> 0 +(check "non-numeric -> 0" (parse-after-seq '("--after" "abc")) 0) +(check "negative -> 0" (parse-after-seq '("--after" "-5")) 0) +(check "2^64 -> 0" (parse-after-seq '("--after" "18446744073709551616")) 0) +(check "u64 max ok" (parse-after-seq '("--after" "18446744073709551615")) + 18446744073709551615) +;; first --after with a value wins +(check "first --after wins" (parse-after-seq '("--after" "7" "--after" "9")) 7) +;; seq 0 is valid +(check "zero seq" (parse-after-seq '("--after" "0")) 0) + +;; ── parse-format: json/human decide; has-db default is quiet ───────────────── +(displayln "parse-format:") +(check "format json" (parse-format '("--format" "json") #f) 'json) +(check "format human" (parse-format '("--format" "human") #f) 'human) +(check "no flag, no db -> human" (parse-format '("watch" "h1") #f) 'human) +(check "no flag, db -> quiet" (parse-format '("watch" "h1") #t) 'quiet) +;; an unknown --format value does NOT consume the value; a later json still wins +(check "unknown then json" (parse-format '("--format" "xml" "--format" "json") #f) 'json) +;; a trailing --format (no value) is ignored; falls to the has-db default +(check "trailing --format, db" (parse-format '("a" "--format") #t) 'quiet) +;; the FIRST decisive --format wins even if a later one disagrees +(check "first decisive wins" (parse-format '("--format" "human" "--format" "json") #f) + 'human) + +;; ── parse-db-path: value of first --db with a successor, else #f ───────────── +(displayln "parse-db-path:") +(check "db path" (parse-db-path '("--db" "events.db")) "events.db") +(check "db mid-list" (parse-db-path '("watch" "h1" "--db" "e.db" "--format" "json")) + "e.db") +(check "no --db -> #f" (parse-db-path '("watch" "h1")) #f) +(check "trailing --db -> #f" (parse-db-path '("watch" "--db")) #f) + +;; ── normalize-host: append default port when no colon ──────────────────────── +(displayln "normalize-host:") +(check "default port" (normalize-host "infra1") "infra1:31337") +(check "explicit port kept" (normalize-host "10.0.0.1:9000") "10.0.0.1:9000") +;; ANY colon counts (Rust host.contains(':')) -> bare ipv6 left as-is +(check "bare ipv6 untouched" (normalize-host "::1") "::1") +(check "default-port export" *default-port* 31337) + +;; ── collect-positional-hosts: skip value flags + their values, drop other -- ─ +(displayln "collect-positional-hosts:") +(check "three hosts normalized" + (collect-positional-hosts '("infra1" "infra2" "infra3")) + '("infra1:31337" "infra2:31337" "infra3:31337")) +;; --db consumes its value; --format consumes its value; positional kept +(check "flags and values skipped" + (collect-positional-hosts '("--db" "events.db" "infra1" "--format" "json" "infra2")) + '("infra1:31337" "infra2:31337")) +;; an unknown --flag (not a value flag) is dropped but consumes nothing +(check "unknown flag dropped, next kept" + (collect-positional-hosts '("--verbose" "infra1")) + '("infra1:31337")) +;; --after consumes its value too +(check "--after value skipped" + (collect-positional-hosts '("--after" "1000" "host:5")) + '("host:5")) +(check "empty -> no hosts" (collect-positional-hosts '()) '()) + +;; ── hosts-from-text: trim, drop blanks/# comments, normalize ───────────────── +(displayln "hosts-from-text:") +(check "lines parsed" + (hosts-from-text "infra1\ninfra2:9000\n") + '("infra1:31337" "infra2:9000")) +;; blank lines and # comments are dropped; surrounding whitespace trimmed +(check "comments and blanks dropped" + (hosts-from-text "# nodes\n\n infra1 \n#skip\ninfra2\n") + '("infra1:31337" "infra2:31337")) +(check "empty text -> no hosts" (hosts-from-text "") '()) + +(newline) +(if (= fails 0) + (displayln "OK: collector-cli matches secmon's collector.rs parse helpers.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/collector-cli.ss @@ -0,0 +1,118 @@ +#!chezscheme +;;; jsecmon collector CLI parse helpers (secmon src/bin/collector.rs), untyped. +;;; +;;; The pure argument/hosts parsing of the `collector` binary, lifted out of the +;;; async networking + key-loading + SQLite side effects (all deferred): +;;; parse-after-seq : args -> u64 (the --after sequence, 0 if absent/bad) +;;; parse-format : args has-db? -> 'json | 'human | 'quiet +;;; parse-db-path : args -> path string | #f +;;; normalize-host : host -> host:PORT (default port if no colon) +;;; collect-positional-hosts : args -> normalized hosts, flag values skipped +;;; hosts-from-text : --hosts-file CONTENTS -> normalized hosts +;;; DEFAULT_PORT is 31337; OutputFormat's three variants map to the symbols +;;; 'human / 'json / 'quiet. +;;; +;;; Faithful corners (mirroring the Rust per-index `for` scans exactly): +;;; * parse-after-seq returns at the FIRST `--after` that HAS a value, taking +;;; `val.parse::<u64>().unwrap_or(0)` (non-numeric / negative / >= 2^64 -> 0); +;;; a trailing `--after` with no value falls through and the result is 0. +;;; * parse-format scans EVERY index: the first `--format` whose value is +;;; "json"/"human" wins, but an UNKNOWN value does NOT skip that value — the +;;; loop simply advances one position (so a later `--format json` can still +;;; win). This differs from analyze.rs's is_json_format, which returns on the +;;; first `--format` that has any value. With no decisive flag: has_db -> +;;; 'quiet (DB storage suppresses stdout), else 'human. +;;; * parse-db-path returns the value of the FIRST `--db` that has a successor. +;;; * normalize-host keys off ANY ':' (Rust `host.contains(':')`), so a bare +;;; IPv6 literal is treated as already-ported — faithful to secmon. +;;; * collect-positional-hosts skips the four value-taking flags AND their +;;; immediately-following value, drops any other `--`-prefixed arg, and +;;; normalizes every remaining positional. +;;; * hosts-from-text trims each line and drops blanks and `#` comments before +;;; normalizing (the fs::read_to_string and the `--hosts-file` lookup are the +;;; deferred I/O; this is the pure contents->hosts core). +;;; +;;; secmon has no #[test] for these, so examples/collector_cli_check.ss derives +;;; every expectation from the Rust source and IS the spec for this port. + +(library (jsecmon collector-cli) + (export parse-after-seq parse-format parse-db-path + normalize-host collect-positional-hosts hosts-from-text + *default-port*) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?)) + + (def *default-port* 31337) + (def *value-flags* '("--db" "--format" "--hosts-file" "--after")) + + ;; first `--after` with a value -> u64 (unwrap_or 0); trailing one falls + ;; through. parse::<u64> rejects negatives / non-integers / >= 2^64 -> 0. + (def (parse-after-seq args) + (let loop ((rest args)) + (cond + ((null? rest) 0) + ((string=? (car rest) "--after") + (if (pair? (cdr rest)) + (let ((n (string->number (cadr rest)))) + (if (and n (integer? n) (>= n 0) (< n (expt 2 64))) n 0)) + (loop (cdr rest)))) + (#t (loop (cdr rest)))))) + + ;; per-index scan: first `--format` value of "json"/"human" decides; any + ;; other value (or a trailing `--format`) just advances one position. + (def (parse-format args has-db) + (let loop ((rest args)) + (cond + ((null? rest) (if has-db 'quiet 'human)) + ((string=? (car rest) "--format") + (if (pair? (cdr rest)) + (let ((v (cadr rest))) + (cond + ((string=? v "json") 'json) + ((string=? v "human") 'human) + (#t (loop (cdr rest))))) + (loop (cdr rest)))) + (#t (loop (cdr rest)))))) + + ;; value of the first `--db` that has a successor; trailing `--db` -> #f. + (def (parse-db-path args) + (let loop ((rest args)) + (cond + ((null? rest) #f) + ((string=? (car rest) "--db") + (if (pair? (cdr rest)) (cadr rest) (loop (cdr rest)))) + (#t (loop (cdr rest)))))) + + (def (normalize-host host) + (if (string-contains host ":") host (str host ":" *default-port*))) + + ;; positional (non-flag) hosts, skipping the four value flags and their values. + (def (collect-positional-hosts args) + (let loop ((rest args) (skip #f) (acc '())) + (if (null? rest) + (reverse acc) + (let ((arg (car rest))) + (cond + (skip (loop (cdr rest) #f acc)) + ((member arg *value-flags*) (loop (cdr rest) #t acc)) + ((not (string-prefix? "--" arg)) + (loop (cdr rest) #f (cons (normalize-host arg) acc))) + (#t (loop (cdr rest) #f acc))))))) + + ;; pure contents->hosts core of parse_hosts_file (the fs read is deferred I/O). + (def (hosts-from-text text) + (filter-map + (lambda (l) + (let ((t (string-trim l))) + (and (not (string-empty? t)) + (not (string-prefix? "#" t)) + (normalize-host t)))) + (string-split text #\newline))))