Complete Jerboa agent polling port
ober
bb1b3844b1e3113af603504a015433b11522223e
--- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ # Generated Rust crate (regenerate with `make rust`). /build/ + +# Compiled binaries (rebuild with `make binaries`). +/jsecmon-keygen +/jsecmon-analyze +/jsecmon-collector +/jsecmon-agent + +# Stray Chez compile artifacts (build-binary.ss cleans these; ignore as backstop). +*.so +*.wpo --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-dns-check monitor-manager-check event-json-check collector-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-auth-check monitor-kernel-check monitor-cron-check monitor-container-check monitor-rootkit-check monitor-podman-check monitor-selinux-check monitor-lateral-check monitor-webshell-check monitor-revshell-check monitor-persistence-check monitor-logtamper-check monitor-dns-check monitor-manager-check event-json-check collector-check protocol-check event-codec-check collector-pull-check agent-server-check checks native-runtime keygen analyze collector agent binaries clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -271,6 +271,33 @@ ioc-check: frame-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/frame_check.ss +# Collector↔agent message codec (secmon src/server/protocol.rs): the bincode 1.x +# body (u32 LE enum tags, fixint LE u64/i64, [u8;32] raw arrays, u64 LE Vec/String +# lengths) composed with frame.ss into ProtocolMessage to_bytes/from_bytes. Pinned +# byte-exact against golden vectors emitted by secmon's own bincode. Pure, no lib. +protocol-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/protocol_check.ss + +# SecurityEvent bincode codec (secmon src/monitor/events.rs to_bytes/from_bytes): +# the per-variant encode/decode the agent & collector share, with u16/i32 and the +# 1-byte Option tag bincode adds over the protocol layer. Pinned byte-exact vs +# secmon golden vectors + round-trips every agent-emitted variant. Pure, no lib. +event-codec-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_codec_check.ss + +# End-to-end collector pull (no socket): the exact byte pipeline bin/collector.ss +# drives -- ECIES + PSK-transport + framed bincode + SecurityEvent codec compose, +# plus the handshake leg and a wrong-key negative. Uses the rust crypto kernels. +collector-pull-check: rust + cd $(BUILD) && cargo build --release + $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_pull_check.ss + +# Agent-side poll server: PSK challenge verification, request dispatch, ECIES +# buffer storage, and a real loopback collector-style socket exchange. +agent-server-check: rust + cd $(BUILD) && cargo build --release + $(SCHEME) --libdirs $(LIBDIRS) --script examples/agent_server_check.ss + # Cross-event correlation rules (secmon storage/mod.rs detect_*), pure cores over # pre-shaped rows. Batch 1 GROUP-BY: brute_force/credential_stuffing (≥5 / 10min), # dns_tunnel (≥50 / 5min), suspicious_cron (non-root). Batch 2 windows/pairs: @@ -363,12 +390,72 @@ monitor-network-check: monitor-files-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_files_check.ss +# Auth monitor (secmon auth.rs): baseline-at-EOF, auth-log tailing, rotation, +# parser-to-event shaping, and utmp session Login/Logout deltas. Pure fixtures. +monitor-auth-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_auth_check.ss + +# Kernel-module monitor (secmon kernel.rs): baseline current modules and emit +# load/unload events from the provider's module list. Pure fixtures. +monitor-kernel-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_kernel_check.ss + +# Scheduled-task monitor loop (secmon cron.rs): baseline and diff task files, +# emitting Created/Modified/Deleted ScheduledTaskChange rows. Pure fixtures. +monitor-cron-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_cron_loop_check.ss + +# Container escape monitor (secmon container.rs): isolated-only scan over new +# mounts, dangerous paths, privileged mode, and dangerous capabilities. +monitor-container-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_container_check.ss + +# Rootkit monitor loop (secmon rootkit.rs): hidden PID and /proc anomaly rows, +# emitted as ContainerEvent/NamespaceEscape like the Rust monitor. +monitor-rootkit-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_rootkit_check.ss + +# Podman polling monitor (secmon podman.rs fallback): baseline container set, +# emit start/remove deltas, and carry security-relevant container fields. +monitor-podman-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_podman_check.ss + +# SELinux monitor loop (secmon selinux.rs): baseline audit log/mode, then emit +# mode changes and parsed audit events from the provider tail. +monitor-selinux-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_selinux_loop_check.ss + +# Lateral movement monitor (secmon lateral.rs): internal SSH/RDP/WinRM/SMB +# connection alerts plus per-target port-scan tracking. +monitor-lateral-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_lateral_check.ss + +# Webshell monitor loop (secmon webshell.rs): web server parent/grandparent +# process scan with deduped suspicious child spawns. +monitor-webshell-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_webshell_loop_check.ss + +# Reverse-shell monitor loop (secmon revshell.rs): connection and cmdline scans +# with per-(pid,remote,port) deduping. +monitor-revshell-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_revshell_check.ss + +# Persistence monitor loop (secmon persistence.rs): baseline persistence files, +# emit created/modified/deleted PersistenceEvent rows with suspicious snippets. +monitor-persistence-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_persistence_check.ss + +# Log tamper monitor loop (secmon logtamper.rs): baseline sizes/mtimes and emit +# truncation/history-clear/timestamp/deleted LogTamperEvent rows. +monitor-logtamper-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_logtamper_check.ss + # DNS monitor connection-polling fallback (secmon dns.rs): port-53 filter, 5 s # dedup window, 30 s cleanup — pure over the shared net-provider seam. No dylib. monitor-dns-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_dns_check.ss -# Monitor manager (secmon mod.rs): boot + one poll cycle running all four +# Monitor manager (secmon mod.rs): boot + one poll cycle running the polling # monitors into a single merged event stream, over fixture providers. No dylib. monitor-manager-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_manager_check.ss @@ -422,6 +509,10 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_summary_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/ioc_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/frame_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/protocol_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_codec_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_pull_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/agent_server_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/correlate_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/revshell_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/cron_check.ss @@ -437,10 +528,64 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_process_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_network_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_files_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_auth_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_kernel_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_cron_loop_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_container_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_rootkit_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_podman_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_selinux_loop_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_lateral_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_webshell_loop_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_revshell_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_persistence_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_logtamper_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_dns_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_manager_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_json_check.ss $(LOADER_ENV) $(SCHEME) --libdirs $(LIBDIRS) --script examples/collector_check.ss +# ── Native binaries ─────────────────────────────────────────────────────────── +# The shippable artifacts are compiled, self-contained executables — never +# `scheme --script` launchers. build-binary.ss whole-program-compiles a bin/*.ss +# entry point, bundles the Chez kernel + stdlib boot image, and STATICALLY links +# the Rust kernels from libjerboa_typed_generated.a (force_load + export_dynamic, +# so the jt_* C-ABI symbols resolve in-process via dlsym). The result needs no +# scheme, no .ss, and no jsecmon kernel .dylib at runtime. SQLite-using binaries +# preload Jerboa's native dylib by absolute path and register its SQLite FFI +# symbols, so they do not need DYLD_LIBRARY_PATH/LD_LIBRARY_PATH. +native-runtime: + cd $(JERBOA)/jerboa-native-rs && cargo build --release --features full + $(MAKE) -C $(JERBOA) native + +keygen: rust + cd $(BUILD) && cargo build --release + JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/keygen.ss jsecmon-keygen + +# secmon-analyze: query/detect/triage/risk over the SQLite store. Statically +# links the Rust kernels (lolbin/dga scoring, calendar) like keygen; SQLite is +# provided by the preloaded Jerboa native runtime. +analyze: rust native-runtime + cd $(BUILD) && cargo build --release + JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/analyze.ss jsecmon-analyze + +# secmon-collector: pull/status/watch over the PSK-encrypted protocol; decrypts +# ECIES events and prints (human/NDJSON) and/or stores them. Statically links the +# Rust crypto kernels (ECIES/PSK/HKDF/AES-GCM); libc sockets resolve in-process. +# SQLite is available through the same preloaded native runtime when --db is used. +collector: rust native-runtime + cd $(BUILD) && cargo build --release + JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/collector.ss jsecmon-collector + +# secmon-agent: Linux polling monitors + encrypted event buffer + poll server. +# It loads the collector public key and PSK at runtime, never embeds secrets. +agent: rust native-runtime + cd $(BUILD) && cargo build --release + JERBOA_HOME="$(JERBOA)" $(SCHEME) --libdirs $(LIBDIRS) --script build-binary.ss bin/agent.ss jsecmon-agent + +# All shippable binaries. +binaries: keygen analyze collector agent + clean: rm -rf $(BUILD) + rm -f jsecmon-keygen jsecmon-analyze jsecmon-collector jsecmon-agent --- a/README.md +++ b/README.md @@ -66,9 +66,15 @@ make monitor-process-check # process monitor scan loop: pid diff/classify/emit o make monitor-network-check # network monitor scan loop: listener/connection dedup+classify over fixture make monitor-files-check # file integrity monitor: detect-change priority + new-file discovery make monitor-dns-check # dns monitor connection-polling fallback: port-53 filter + 5 s dedup window -make monitor-manager-check # monitor manager: boot + one poll cycle through all four monitors (fixture) -make event-json-check # event JSON serializer: all 11 variants round-tripped and field-asserted +make monitor-podman-check # podman polling fallback: start/remove container deltas +make monitor-selinux-check # SELinux monitor loop: mode changes + tailed audit events +make monitor-lateral-check # lateral monitor loop: internal service use + port scan tracking +make monitor-webshell-check # webshell monitor loop: suspicious web-server descendants +make monitor-revshell-check # reverse-shell monitor loop: connection + cmdline signals +make monitor-manager-check # monitor manager: boot + one full polling cycle (fixture) +make event-json-check # event JSON serializer: every Rust EventType variant make collector-check # agent collector loop: boot+tick into a real SQLite store, rows queried back +make agent-server-check # agent poll server: PSK auth + request loop over a real TCP socket make checks # every Jerboa-side check in one shot ``` @@ -96,6 +102,31 @@ wrappers (`make kernels-check`). `make` needs a built jerboa checkout at `$JERBOA` (default `~/mine/jerboa`), whose `.chez/bin/scheme` and `support/typed-rust.ss` drive the backend. +## Binaries + +The shippable artifacts are **compiled, self-contained executables** — never +`scheme --script foo.ss` launchers. `build-binary.ss` takes a `bin/*.ss` entry +point, whole-program-compiles it (any stdlib library that ships only a `.so`, +e.g. `(jerboa core)`, comes back in `compile-whole-program`'s missing list and +is bundled into the boot image instead of inlined), bundles the Chez kernel + +stdlib boot, and **statically links** the Rust kernels from +`libjerboa_typed_generated.a` — `-force_load` pulls the archive in whole (the C +`main` never names the `jt_*` symbols; Chez resolves them at runtime via +`dlsym`) and `-export_dynamic` makes them visible to that `dlsym`; `kernels.ss` +does `(load-shared-object #f)` in this mode. The result is one Mach-O/ELF file +that needs no scheme, no `.ss`, and no kernel `.dylib` at runtime. + +`make keygen` builds `jsecmon-keygen` (the ECIES keypair + PSK generator, port of +`secmon-keygen`); `make binaries` builds all of them. The `make *-check` vector +tests stay as dev-time `.ss` scripts (the test harness, not shipped). + +| secmon binary | jsecmon | status | +|------------------|--------------------|---------------------------------| +| `bin/keygen` | `bin/keygen.ss` → `make keygen` | ✅ **compiled binary** — proves the build pipeline (WPO → boot image → static-FFI link). Self-contained native executable; output matches `secmon-keygen` byte-for-byte (ECIES pub/priv + 32-byte PSK as lowercase hex from the OS CSPRNG via the x25519 + hex kernels). Verified to run standalone with the kernel dylib off every search path. | +| `bin/analyze` | `bin/analyze.ss` → `make analyze` | ✅ **compiled binary** — assembles `run_detections` itself over `(jsecmon storage)`; summary/query/anomalies/first-seen/timeline/retention all wired to the ported detection + analytics layers. | +| `bin/collector` | `bin/collector.ss` → `make collector` | ✅ **compiled binary** — the agent **pull** client: PSK handshake (Challenge → ChallengeResponse) over the transport envelope, then `get_events_after`/`status` requests; each `SerializedEvent` is ECIES-decrypted to a `SecurityEvent` and printed (`--format human`/`json`, secmon-faithful) or persisted to SQLite (`--db`, advancing `collector_state`). `poll`/`status`/`watch` subcommands. Verified end-to-end over a real TCP socket (`examples/fake_agent.ss` loopback) and byte-wise without a socket (`make collector-pull-check`). Loads keys from env/file at runtime, never embedded. | +| `bin/agent` | `bin/agent.ss` → `make agent` | ✅ **compiled binary** — Linux polling-agent shell: loads the collector public key + PSK from env/file at runtime, starts the PSK-authenticated pull server, ECIES-encrypts `SecurityEvent` bytes into the priority buffer, emits `agent_start`/`heartbeat`, and drains the full provider-backed polling set (`process`, `network`, `files`, `auth`, `kernel`, `scheduled`, `container`, `rootkit`, `podman`, `selinux`, `persistence`, `lateral`, `logtamper`, `webshell`, `revshell`, `dns`) when `/proc` is available. Verified with `jsecmon-collector status`/`poll` against the compiled agent on a real loopback TCP socket. | + ## Port status Driven by what each module needs from the backend; pure-logic detection first, @@ -168,11 +199,13 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `monitor/lateral` IP parsing + internal-network test (`is_internal_ip`) | `jsecmon/ipaddr.ss` | ✅ **untyped layer** — `is_internal_ip` parses `ip_str.parse::<IpAddr>()` first and only classifies on success, so `parse-ipv4` / `parse-ipv6` faithfully reproduce **Rust std's `IpAddr` FromStr** boundary: IPv4 = exactly 4 octets, 1–3 digits, **no leading zeros**, ≤255; IPv6 = colon-separated 1–4-digit hex groups with at most one `::` (eliding ≥1 zero group) and an optional trailing embedded IPv4 (forbidden before `::`). `is-internal-ip` then mirrors lateral.rs: V4 `10/8` · `172.16/12` · `192.168/16` · `127/8`, V6 `fc00::/7` (`seg0 & 0xfe00 == 0xfc00`) or loopback. lateral.rs has no #[test]; expectations were generated by a **std-only Rust oracle** over ~70 inputs, so `make ipaddr-check` (88 cases) pins the port to real Rust and **is** the spec. | | `monitor/auth` log-line parsers (`parse_auth_line` + 7 sub-parsers) | `jsecmon/auth.ss` | ✅ **untyped layer** — the dispatcher lowercases and routes on substrings (`sshd`→ssh, `sudo`, `su[`/`su:`, `authentication failure`→pam, `useradd`/`adduser`, `userdel`/`deluser`, `password changed`/`passwd`), each sub-parser pulling fields with `extract_field(line, start, end)`. Faithful corners: `extract_field` searches the **raw** line (case-sensitive) while dispatch/success use the lower-cased copy, so `Invalid user admin` → username `unknown`; the SSH "accepted" branch `?`-propagates a missing `for ` to **#f** (no fall-through); and the documented quirks where the username captures `TTY=pts/0` / `authentication failure; TTY=…` / `root)` are preserved. AuthEventType → `'ssh-key-auth 'login 'failed-login 'sudo-attempt 'su-attempt 'user-created 'user-deleted 'password-change`; None → `#f`. auth.rs has no #[test]; `make auth-check` (24 cases) derives every expectation from a **std-only Rust oracle** over the verbatim bodies and **is** the spec. | | `format_ts` / `format_ts_iso` / `parse_hour_to_ms` / `parse_datetime` (chrono UTC calendar) | `jsecmon/calendar.ss` | ✅ **untyped layer** — the pure epoch-ms ↔ civil-date boundary the detect/analytics seeds defer to. secmon formats/parses timestamps with chrono always in UTC (`DateTime::from_timestamp_millis`, `NaiveDateTime`), which is pure proleptic-Gregorian arithmetic, reproduced exactly with Howard Hinnant's `days_from_civil`/`civil_from_days`: `format-ts` (`%Y-%m-%d %H:%M:%S`, storage+analyze), `format-ts-iso` (`%Y-%m-%dT%H:%M:%S%.3fZ`, analyze), `parse-hour-to-ms` (`%Y-%m-%d %H`+`:00:00`, the frequency-spike inverse of `%H` decomposition, bad input → 0), and `parse-datetime-ms` (returns the prelude Result: relative `Nh`/`Nd` from an optional `at`/now, then RFC3339 with `Z`/`±HH:MM` offset and optional `.fff`, then naive ISO assumed-UTC, then date-only; else `err` with the exact Rust message). Faithful corners: Hinnant's `/` truncates toward zero (`quotient`) but the ms→day split must **floor** (a pre-1970 ms keeps a non-negative ms-of-day), so the parts use exact `floor`; the export is `parse-datetime-ms`/`char-index` to avoid shadowing the prelude's own `parse-datetime` (a datetime record) and `string-index`. `make calendar-check` pins the civil round-trip, both formatters (incl. leap-day, negative-ms flooring), `parse_hour_to_ms`, and secmon's `test_parse_datetime` invariants (28 cases). | -| `monitor::process::ProcessMonitor` scan loop | `jsecmon/monitor-process.ss` | ✅ **untyped layer** — secmon's process-monitor scan in pure Jerboa over an injected `mon-provider` seam (list-pids / get-process / hostname), fixture-tested: diff pid sets → classify each new spawn (check-suspicious delegates to `(jsecmon suspicious)`) → emit process_start/suspicious_exec; exited pids → process_exit; list-pids error → no-op. Faithfully preserves secmon's `continue` before insert (a failed get-process is not added to known and is retried), the parent-cache lookup, and the 1000-entry cache prune. `make-linux-provider` is the thin /proc shell via `(jsecmon proc-linux)` parsers (exe/cwd readlink deferred). `make monitor-process-check` (30 cases). | -| `monitor::network::NetworkMonitor` scan loop | `jsecmon/monitor-network.ss` | ✅ **untyped layer** — the network-monitor scan over an injected `net-provider` seam: listeners → listening_port/info (de-dup key `proto:laddr:lport`); connections → `(jsecmon netconn)` connection-suspicious? → suspicious_connection/high or network_connection/info (de-dup key `proto:laddr:lport:raddr:rport:state`). Provider Err skips that half; never ages out (secmon no-op TODO). `make-linux-network-provider` reads /proc/net/{tcp,udp,tcp6,udp6} via proc-linux parsers (socket-inode→pid walk deferred). `make monitor-network-check` (13 cases). | -| `monitor::files::FileIntegrityMonitor` scan loop | `jsecmon/monitor-files.ss` | ✅ **untyped layer** — file-integrity monitor over an injected `file-provider` seam: `baseline-files` records initial state; `detect-change` produces the ordered diff (deleted > created > permission-changed > owner-changed > modified) — pinned secmon quirk: owner-changed reports `old_mode = new_mode = new.mode`; `scan-files` re-stats tracked paths + discovers new files in watched dirs (Created), classifying via `(jsecmon file-change)` is-suspicious-change. `make-linux-file-provider` uses file-exists?/file-modification-time (stat+sha256 FFI deferred); `linux-monitored-paths` = secmon's Linux PLATFORM_PATHS. `make monitor-files-check` (18 cases). | +| `monitor::process::ProcessMonitor` scan loop | `jsecmon/monitor-process.ss` | ✅ **untyped layer** — secmon's process-monitor scan in pure Jerboa over an injected `mon-provider` seam (list-pids / get-process / hostname), fixture-tested: diff pid sets → classify each new spawn (check-suspicious delegates to `(jsecmon suspicious)`) → emit process_start/suspicious_exec; exited pids → process_exit; list-pids error → no-op. Faithfully preserves secmon's `continue` before insert (a failed get-process is not added to known and is retried), the parent-cache lookup, and the 1000-entry cache prune. `make-linux-provider` is the thin /proc shell via `(jsecmon proc-linux)` parsers plus live `/proc/[pid]/{exe,cwd}` readlinks. `make monitor-process-check` (30 cases). | +| `monitor::network::NetworkMonitor` scan loop | `jsecmon/monitor-network.ss` | ✅ **untyped layer** — the network-monitor scan over an injected `net-provider` seam: listeners → listening_port/info (de-dup key `proto:laddr:lport`); connections → `(jsecmon netconn)` connection-suspicious? → suspicious_connection/high or network_connection/info (de-dup key `proto:laddr:lport:raddr:rport:state`). Provider Err skips that half; never ages out (secmon no-op TODO). `make-linux-network-provider` reads /proc/net/{tcp,udp,tcp6,udp6} via proc-linux parsers and resolves socket inode → `(pid . process-name)` by scanning `/proc/*/fd`, enabling the live web-server outbound check. `make monitor-network-check` (13 cases). | +| `monitor::files::FileIntegrityMonitor` scan loop | `jsecmon/monitor-files.ss` | ✅ **untyped layer** — file-integrity monitor over an injected `file-provider` seam: `baseline-files` records initial state; `detect-change` produces the ordered diff (deleted > created > permission-changed > owner-changed > modified) — pinned secmon quirk: owner-changed reports `old_mode = new_mode = new.mode`; `scan-files` re-stats tracked paths + discovers new files in watched dirs (Created), classifying via `(jsecmon file-change)` is-suspicious-change. `make-linux-file-provider` now uses POSIX stat mode/uid/gid/mtime and SHA-256 for regular files under 10 MiB; `linux-monitored-paths` = secmon's Linux PLATFORM_PATHS. `make monitor-files-check` (18 cases). | | `monitor::dns::DnsMonitor` connection-polling fallback | `jsecmon/monitor-dns.ss` | ✅ **untyped layer** — the polling fallback: reuse net-provider's list-connections; keep only remote-port==53; dedup key `pid:remote_addr:protocol:local_port` (pid defaults to 0); skip if seen <5000 ms ago; emit dns_query/info (query_name `"<unknown>"`, query_type UDP/TCP, server_addr=remote-addr, response_addrs=[]). `cleanup-dns-queries` ages entries ≥30 s. Shares net-provider with the network monitor exactly as secmon. `make monitor-dns-check` (14 cases). | -| `monitor::mod::MonitorManager` (boot + poll cycle) | `jsecmon/monitor-manager.ss` | ✅ **untyped layer** — `monitor-boot` baselines files + emits agent_start; `monitor-tick` runs process/network/files/dns in fixed order and returns the merged event stream; cleanup-dns-queries runs after tick. `make-linux-monitor-set` wires all four live Linux providers. `make monitor-manager-check` drives the full four-monitor cycle end to end with fixture providers. | -| `event_json.rs get_event_json_data` | `jsecmon/event-json.ss` | ✅ **untyped layer** — the flat JSON `data` body for all 11 event variants, matching secmon's json!{...} field SETS exactly: suspicious_exec drops cwd/name; listening_port keeps only local side; suspicious_connection drops state/process_name; suspicious_file_change keeps reason+hashes only. Option::None → JSON null via `(if #f #f)` void sentinel; parent process → nested {pid,name,exe} or null; change_type = Rust Debug form ("Modified", "PermissionChanged", …); dns_query always carries response_addrs=[]. `make event-json-check` drives real monitor events through the serializer, parses back, and asserts all tricky corners. | -| `bin/agent` storage path (collector loop) | `jsecmon/collector.ss` | ✅ **untyped layer** — ties monitor runtime to storage: `collect-event!` derives the full storage row (data = event-data-json, summary = build-summary, pid/pname = extract-pid/extract-process-name), assigns a monotonic seq, INSERT-OR-IGNORE dedup; `collect-events!` drains a batch; `run-boot!`/`run-tick!` tie monitor-boot/monitor-tick to the store. `run-agent` is the thin live shell (real ms clock, Linux monitor-set, poll/sleep). `make collector-check` drives boot+tick of the four-monitor fixture set into a real temp SQLite store, queries rows back to assert event count, derived pid/name/summary, parsed data, and dedup (19 cases). | -| server listener / ebpf / dtrace I/O / live FFI gaps | — | ⏳ I/O+async+FFI, last — server pull protocol (ProtocolMessage codec + request handler); live-shell FFI gaps (process exe/cwd readlink, network socket-inode→pid walk, file stat+sha256); remaining monitor loops (rootkit/persistence/cron/auth/container/lateral/revshell — classifiers done, enumeration shells deferred); ebpf/dtrace AF_PACKET capture; EncryptedPayload bincode framing (only if wire-compat needed). | +| `monitor::mod::MonitorManager` (boot + poll cycle) | `jsecmon/monitor-manager.ss` | ✅ **untyped layer** — `monitor-boot` baselines stateful monitors and emits agent_start; `monitor-tick` runs the full polling set in a fixed order and returns the merged event stream; cleanup-dns-queries runs after tick. `make-linux-monitor-set` wires the live Linux providers, sharing process/network providers where the Rust monitors do. `make monitor-manager-check` drives the full cycle end to end with fixture providers. | +| `event_json.rs get_event_json_data` | `jsecmon/event-json.ss` | ✅ **untyped layer** — the flat JSON `data` body for every Rust `EventType` variant, matching secmon's json!{...} field SETS exactly: suspicious_exec drops cwd/name; listening_port keeps only local side; suspicious_connection drops state/process_name; mount_event drops target/fstype but computes `dangerous`; namespace/capability derive `ns_types`/`dangerous_capabilities`; all enum-ish fields use Rust Debug spelling. Option::None → JSON null via `(if #f #f)` void sentinel; parent process → nested {pid,name,exe} or null. `make event-json-check` drives monitor events plus representative eBPF/podman/SELinux/revshell/lateral/webshell rows through the serializer, parses back, and asserts the tricky corners. | +| `monitor::events::SecurityEvent` bincode | `jsecmon/event-codec.ss` | ✅ **untyped layer** — byte-exact bincode reader/writer for the top-level `SecurityEvent` and all 32 `EventType` discriminants, with golden secmon bytes for the process/network base cases and round-trip coverage for every remaining variant. This is the compiled agent's event payload codec feeding the encrypted buffer and collector pull path. | +| `bin/agent` storage path (collector loop) | `jsecmon/collector.ss` | ✅ **untyped layer** — ties monitor runtime to storage: `collect-event!` derives the full storage row (data = event-data-json, summary = build-summary, pid/pname = extract-pid/extract-process-name), assigns a monotonic seq, INSERT-OR-IGNORE dedup; `collect-events!` drains a batch; `run-boot!`/`run-tick!` tie monitor-boot/monitor-tick to the store. `run-agent` is the thin live shell (real ms clock, Linux monitor-set, poll/sleep). `make collector-check` drives boot+tick of the full fixture monitor set into a real temp SQLite store, queries rows back to assert event count, derived pid/name/summary, parsed data, and dedup. | +| `server::PollServer` | `jsecmon/agent-server.ss` | ✅ **untyped layer** — server-side pull protocol: creates PSK challenge, verifies ChallengeResponse freshness/proof, handles `get_events_after`, `get_events_in_range`, `status`, `acknowledge`, and `ping`, and serializes ECIES-sealed `SerializedEvent`s from the priority buffer. `make agent-server-check` covers direct buffer decrypt plus a real TCP loopback request sequence. | +| ebpf / dtrace I/O / remaining platform shells | — | ⏳ deferred high-fidelity capture — the provider-backed polling agent path is wired; remaining parity is eBPF/DTrace syscall capture for file-open/ptrace/namespace/mount/capability events, AF_PACKET DNS capture, Podman event-stream mode (polling fallback is ported), local encrypted store parity, and Rust `EncryptedPayload` bincode framing only if wire compatibility with the Rust agent is required. | new file mode 100644 --- /dev/null +++ b/bin/agent.ss @@ -0,0 +1,199 @@ +#!chezscheme +;;; jsecmon-agent — monitor locally, ECIES-buffer events, serve collector pulls. + +(import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (only (jsecmon config) config-from-env) + (only (jsecmon kernels) hex-decode) + (only (jsecmon agent-server) + make-agent-runtime agent-store-event! agent-store-events! + agent-buffered-count agent-uptime-secs + agent-server-start! agent-server-port) + (only (jsecmon monitor-manager) + make-linux-monitor-set monitor-boot monitor-tick monitor-set-hostname) + (only (jsecmon calendar) now-ms)) + +(def *version* "jsecmon-jerboa") +(def *heartbeat-interval-ms* 30000) + +(def (println . parts) (for-each display parts) (newline)) +(def (eprintln . parts) + (let ((p (current-error-port))) + (for-each (lambda (x) (display x p)) parts) + (newline p))) +(def (die . parts) (apply eprintln parts) (exit 1)) + +(def (debug?) (let ((v (getenv "SECMON_DEBUG"))) (and v (not (string=? v "")) (not (string=? v "0"))))) +(def (arg? flag argv) (and (member flag argv) #t)) + +(def (usage) + (eprintln "jsecmon-agent - Monitor this host and serve encrypted events\n") + (eprintln "Usage:") + (eprintln " jsecmon-agent [--no-monitors]\n") + (eprintln "Environment variables:") + (eprintln " SECMON_PUBLIC_KEY ECIES public key (hex) or path to key file") + (eprintln " SECMON_AGENT_PUBLIC Alias for SECMON_PUBLIC_KEY") + (eprintln " ECIES_PUBLIC_KEY Alias accepted from keygen output") + (eprintln " SECMON_PSK Pre-shared key (hex) or path to key file") + (eprintln " PSK Alias accepted from keygen output") + (eprintln " SECMON_LISTEN Listen address, default 0.0.0.0:31337") + (eprintln " SECMON_POLL_MS Monitor poll interval, default 100") + (eprintln " SECMON_BUFFER_SIZE Encrypted event buffer cap, default 10000") + (eprintln " SECMON_DEBUG=1 Print startup diagnostics")) + +(def (cfg-ref cfg key) (cdr (assq key cfg))) + +(def (read-file-trimmed path) + (string-trim (call-with-input-file path get-string-all))) + +(def (lookup-env names) + (let loop ((ns names)) + (cond ((null? ns) #f) + ((getenv (car ns)) => (lambda (v) v)) + (#t (loop (cdr ns)))))) + +(def (path-like-key? v) + (and (< (string-length v) 128) + (or (string-prefix? "/" v) + (string-prefix? "." v) + (file-exists? v)))) + +(def (resolve-key-value v) + (if (path-like-key? v) + (try (read-file-trimmed v) + (catch (e) (die "failed to read key file " v ": " e))) + v)) + +(def (first-existing paths) + (let loop ((ps paths)) + (cond ((null? ps) #f) + ((file-exists? (car ps)) (car ps)) + (#t (loop (cdr ps)))))) + +(def (load-key-hex label env-names file-env-names default-paths) + (cond + ((lookup-env env-names) => resolve-key-value) + ((lookup-env file-env-names) + => (lambda (p) + (try (read-file-trimmed p) + (catch (e) (die "failed to read " label " file " p ": " e))))) + ((first-existing default-paths) + => (lambda (p) (read-file-trimmed p))) + (else + (die label " not set (export hex or point an env var at a key file)")))) + +(def (decode-32 label hex) + (try + (let ((b (hex-decode hex))) + (unless (= (bytevector-length b) 32) + (die label " must decode to exactly 32 bytes")) + b) + (catch (e) (die "invalid " label ": " e)))) + +(def (last-colon s) + (let loop ((i (- (string-length s) 1))) + (cond ((< i 0) #f) + ((char=? (string-ref s i) #\:) i) + (#t (loop (- i 1)))))) + +(def (split-host-port hp) + (let ((i (last-colon hp))) + (if i + (values (substring hp 0 i) + (or (string->number (substring hp (+ i 1) (string-length hp))) 31337)) + (values hp 31337)))) + +(def (fallback-hostname) + (or (getenv "HOSTNAME") + (guard (e (#t "unknown")) + (let ((h (read-file-string "/proc/sys/kernel/hostname"))) + (if h (string-trim h) "unknown"))))) + +(def (make-agent-start host version ts) + (let ((h (make-hash-table))) + (hash-put! h "type" "agent_start") + (hash-put! h "severity" "info") + (hash-put! h "host" host) + (hash-put! h "ts" ts) + (hash-put! h "version" version) + h)) + +(def (make-heartbeat rt host) + (let ((h (make-hash-table))) + (hash-put! h "type" "heartbeat") + (hash-put! h "severity" "info") + (hash-put! h "host" host) + (hash-put! h "ts" (now-ms)) + (hash-put! h "uptime_secs" (agent-uptime-secs rt)) + (hash-put! h "events_buffered" (agent-buffered-count rt)) + h)) + +(def (maybe-linux-monitor-set no-monitors?) + (and (not no-monitors?) + (file-directory? "/proc") + (try (make-linux-monitor-set) + (catch (e) + (when (debug?) (eprintln "[debug] monitors unavailable: " e)) + #f)))) + +(def (run-agent-loop rt mset poll-ms host) + (let ((boot-ms (now-ms))) + (if mset + (agent-store-events! rt (monitor-boot mset *version* boot-ms)) + (agent-store-event! rt (make-agent-start host *version* boot-ms)))) + (agent-store-event! rt (make-heartbeat rt host)) + (let loop ((last-heartbeat (now-ms))) + (when mset + (agent-store-events! rt (monitor-tick mset (now-ms)))) + (let ((n (now-ms))) + (if (>= (- n last-heartbeat) *heartbeat-interval-ms*) + (begin + (agent-store-event! rt (make-heartbeat rt host)) + (sleep-ms poll-ms) + (loop n)) + (begin + (sleep-ms poll-ms) + (loop last-heartbeat)))))) + +(def (main) + (let ((argv (command-line-arguments))) + (when (or (arg? "--help" argv) (arg? "-h" argv)) + (usage) + (exit 0)) + (let* ((public-hex (load-key-hex + "public key" + '("SECMON_PUBLIC_KEY" "SECMON_AGENT_PUBLIC" "ECIES_PUBLIC_KEY") + '("SECMON_PUBLIC_KEY_FILE" "SECMON_AGENT_PUBLIC_FILE" "ECIES_PUBLIC_KEY_FILE") + '("keys/public.key" "./keys/public.key"))) + (psk-hex (load-key-hex + "PSK" + '("SECMON_PSK" "PSK") + '("SECMON_PSK_FILE" "PSK_FILE") + '("keys/psk.key" "./keys/psk.key"))) + (public-key (decode-32 "public key" public-hex)) + (psk (decode-32 "PSK" psk-hex)) + (cfg (config-from-env getenv)) + (listen (cfg-ref cfg 'listen-addr)) + (poll-ms (cfg-ref cfg 'poll-interval-ms)) + (max-buffer (cfg-ref cfg 'max-buffer-size))) + (let-values (((host port) (split-host-port listen))) + (let* ((mset (maybe-linux-monitor-set (arg? "--no-monitors" argv))) + (event-host (if mset (monitor-set-hostname mset) (fallback-hostname))) + (rt (make-agent-runtime public-key psk max-buffer event-host)) + (srv (agent-server-start! rt host port))) + (when (debug?) + (eprintln "[debug] jsecmon-agent listening on " host ":" (agent-server-port srv) + " poll_ms=" poll-ms + " buffer=" max-buffer + " monitors=" (if mset "linux" "off"))) + (run-agent-loop rt mset poll-ms event-host)))))) + +(main) new file mode 100644 --- /dev/null +++ b/bin/analyze.ss @@ -0,0 +1,1095 @@ +#!chezscheme +;;; jsecmon secmon-analyze — query & analyze stored security events. +;;; +;;; Port of secmon/src/bin/analyze.rs. The pure helpers (flag scanners, duration +;;; and alert-sink parsing) come from (jsecmon analyze-cli); the data layer is +;;; (jsecmon storage) over SQLite, with detections assembled from (jsecmon +;;; threats)+(jsecmon detect) and ranked/grouped by (jsecmon analytics). +;;; +;;; Detection assembly mirrors secmon's store.run_detections match arms exactly +;;; (see run-all-detections): the 11 SQL/correlation rules from (jsecmon threats), +;;; suspicious_cmdline/dga_domain from (jsecmon detect) over the filtered events, +;;; in DETECTION_RULES order, attack-annotated and time-sorted. detect_anomalies +;;; maps to (jsecmon threats) run-anomaly-detections. The db-layer anomaly hashes +;;; drop secmon's per-finding `description`, so describe-anomaly regenerates it. +;;; +;;; JSON is emitted with object keys SORTED (serde_json BTreeMap, no preserve +;;; order) so it matches secmon byte layout; JSON null is the void singleton, +;;; #f is boolean false, and Option fields (pid/process_name) pass through `jn`. +;;; +;;; Entry point built by build-binary.ss → a standalone native binary; argv comes +;;; in via (command-line-arguments) = (command db remaining…), i.e. secmon's +;;; args[1..]. SQLite still resolves libjerboa_native at runtime via the loader. + +(import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (only (jsecmon storage) + store-open store-close make-filter query-events + store-summary first-seen-after list-hosts + delete-before store-vacuum match-iocs entity-timeline) + (only (jsecmon detect) detect-lolbin-cmdline detect-dga-domain) + (only (jsecmon threats) + detect-brute-force detect-credential-stuffing detect-dns-tunnel + detect-suspicious-cron detect-recon-port-scan detect-data-exfil + detect-priv-escalation-chain detect-persistence-after-access + detect-log-cover detect-lateral-after-shell detect-impossible-travel + run-anomaly-detections) + (only (jsecmon detection-rules) + detection-rules rule-names rule-known? rule-attack anomaly-rule-attack) + (only (jsecmon analytics) + compute-host-risks group-incidents + host-risk-host host-risk-score host-risk-critical host-risk-high + host-risk-medium host-risk-distinct-rules host-risk-chains + host-risk-suspicious-cmdline host-risk-dga host-risk-rootkit-or-tamper + host-risk-persistence host-risk-first-seen-ms host-risk-last-seen-ms + incident-rule incident-host incident-key incident-severity + incident-attack incident-first-ms incident-last-ms + incident-occurrences incident-sample) + (only (jsecmon triage-store) triage-row compute-triaged-ids) + (only (jsecmon triage) + verdict-category verdict-label verdict-headline verdict-explanation) + (only (jsecmon yaml-rules) + load-yaml-rules run-yaml-rule yaml-rule-name yaml-rule-description + yaml-rule-severity yaml-rule-attack) + (only (jsecmon sigma) + convert-sigma-rule imported-rule-yaml imported-rule-name) + (only (jsecmon ioc) parse-ioc-text detect-ioc-type) + (only (jsecmon calendar) format-ts format-ts-iso parse-datetime-ms now-ms) + (only (jsecmon analyze-cli) + parse-duration-ms parse-alert-sink parse-alert-sinks + parse-flag-value has-flag is-json-format) + (only (jsecmon lolbin) score-json-cmdline lol-label-summary)) + +;; ── tiny output helpers ─────────────────────────────────────────────────────── +(def (println . parts) (for-each display parts) (newline)) +(def (eprintln . parts) + (let ((p (current-error-port))) + (for-each (lambda (x) (display x p)) parts) (newline p))) +(def (die . parts) (apply eprintln parts) (exit 1)) + +(def (sev-marker s) + (cond ((string=? s "critical") "!!!") ((string=? s "high") "!! ") + ((string=? s "medium") "! ") (else " "))) + +(def (pad-right x n) + (let* ((s (if (string? x) x (str x))) (k (string-length s))) + (if (>= k n) s (string-append s (make-string (- n k) #\space))))) +(def (pad-left x n) + (let* ((s (if (string? x) x (str x))) (k (string-length s))) + (if (>= k n) s (string-append (make-string (- n k) #\space) s)))) + +;; ── JSON (sorted keys; void→null, #f→false; serde_json byte layout) ─────────── +(def json-null (if #f #f)) ;; the void singleton +(def (jn v) (if (eq? v #f) json-null v)) ;; Option field: #f (NULL) → null + +(def (json-escape s) + (let ((o (open-output-string))) + (string-for-each + (lambda (c) + (let ((n (char->integer c))) + (cond ((char=? c #\") (display "\\\"" o)) + ((char=? c #\\) (display "\\\\" o)) + ((char=? c #\newline) (display "\\n" o)) + ((char=? c #\return) (display "\\r" o)) + ((char=? c #\tab) (display "\\t" o)) + ((= n 8) (display "\\b" o)) + ((= n 12) (display "\\f" o)) + ((< n 32) (display (format "\\u~4,'0x" n) o)) + (else (write-char c o))))) + s) + (get-output-string o))) + +(def (json-num n) + (cond ((and (integer? n) (exact? n)) (number->string n)) + ((and (rational? n) (exact? n)) (number->string (exact->inexact n))) + (else (number->string n)))) + +(def (->json x) + (cond ((eq? x json-null) "null") + ((eq? x #t) "true") + ((eq? x #f) "false") + ((string? x) (string-append "\"" (json-escape x) "\"")) + ((symbol? x) (string-append "\"" (json-escape (symbol->string x)) "\"")) + ((number? x) (json-num x)) + ((hash-table? x) + (let ((ks (list-sort string<? (hash-keys x)))) + (string-append "{" + (string-join + (map (lambda (k) + (string-append "\"" (json-escape k) "\":" (->json (hash-get x k)))) + ks) ",") + "}"))) + ((or (null? x) (pair? x)) + (string-append "[" (string-join (map ->json x) ",") "]")) + (else "null"))) + +(def (obj . kvs) ;; build a string-keyed hash + (let ((h (make-hash-table))) + (let loop ((xs kvs)) + (if (or (null? xs) (null? (cdr xs))) h + (begin (hash-put! h (car xs) (cadr xs)) (loop (cddr xs))))))) + +(def (print-json x) (println (->json x))) + +;; ── filters ─────────────────────────────────────────────────────────────────── +(def (datetime-or-die v flag) + (let ((r (parse-datetime-ms v))) + (if (ok? r) (unwrap r) (die "Invalid " flag " value: " (unwrap-err r))))) + +(def (int-or-die v flag) + (let ((n (string->number v))) + (if (and n (integer? n)) n (die "Invalid " flag " value: " v)))) + +(def (parse-filter args) + (let ((kvs '())) + (def (add! k v) (set! kvs (cons k (cons v kvs)))) ;; reversed pairs; ok for make-filter + (awhen (parse-flag-value args "--host") (add! "host" it)) + (awhen (parse-flag-value args "--type") (add! "event_type" it)) + (awhen (parse-flag-value args "--severity") (add! "severity" it)) + (awhen (parse-flag-value args "--since") (add! "since_ms" (datetime-or-die it "--since"))) + (awhen (parse-flag-value args "--until") (add! "until_ms" (datetime-or-die it "--until"))) + (awhen (parse-flag-value args "--pid") (add! "pid" (int-or-die it "--pid"))) + (awhen (parse-flag-value args "--process") (add! "process_name" it)) + (awhen (parse-flag-value args "--search") (add! "search" it)) + (awhen (parse-flag-value args "--limit") (add! "limit" (int-or-die it "--limit"))) + ;; kvs is (k v k v …) reversed in pairs but make-filter only reads keys + (apply make-filter (let unzip ((xs kvs) (out '())) + (if (null? xs) out (unzip (cddr xs) (cons (car xs) (cons (cadr xs) out)))))))) + +(def (filter-set base k v) ;; clone a filter hash with one key set + (let ((g (make-hash-table))) + (for-each (lambda (key) (hash-put! g key (hash-get base key))) (hash-keys base)) + (hash-put! g k v) g)) + +;; ── per-finding description (db anomaly hashes drop secmon's Anomaly.description) ─ +(def (hour-bucket ts) ;; ms → "%Y-%m-%d %H:00" + (string-append (substring (format-ts ts) 0 13) ":00")) + +(def (fmt1 x) ;; one decimal place, like Rust {:.1} + (let* ((i (exact (round (* (exact->inexact x) 10)))) + (whole (quotient i 10)) (frac (abs (remainder i 10)))) + (str whole "." frac))) + +(def (describe-anomaly a) + (let* ((rule (hash-get a "rule")) (host (hash-get a "host")) + (d (hash-get a "details")) (ts (hash-get a "timestamp_ms")) + (g (lambda (k) (hash-get d k)))) + (cond + ((string=? rule "brute_force") + (str (g "failure_count") " auth failures for '" (g "username") "' on " host " in 10min")) + ((string=? rule "credential_stuffing") + (str (g "distinct_usernames") " distinct users failing from " (g "remote_host") " on " host " in 10min")) + ((string=? rule "dns_tunnel") + (str (g "query_count") " DNS queries from '" (g "process_name") "' on " host " in 5min")) + ((string=? rule "suspicious_cron") + (str "Cron change by non-root user '" (g "user") "' on " host ": " (g "summary"))) + ((string=? rule "recon_port_scan") + (str "'" (g "process_name") "' connected to " (g "distinct_ports") " distinct ports in 5min on " host)) + ((string=? rule "data_exfil") + (str "'" (g "process_name") "' made " (g "connection_count") " outbound connections in 5min on " host)) + ((string=? rule "priv_escalation_chain") + (str "Auth success followed by privilege escalation on " host)) + ((string=? rule "persistence_after_access") + (str (if (equal? (g "event_a") "webshell") "Webshell" "Reverse shell") + " followed by persistence on " host)) + ((string=? rule "lateral_after_shell") + (str "Reverse shell followed by lateral movement on " host)) + ((string=? rule "log_cover") + (str "Critical " (g "trigger_event") " event followed by log tampering on " host)) + ((string=? rule "suspicious_cmdline") + (str "'" (g "process_name") "' on " host " matched LOLBin patterns [" + (lol-label-summary (score-json-cmdline (or (g "cmdline") "") (or (g "exe") ""))) + "] (score " (g "score") ")")) + ((string=? rule "dga_domain") + (str "'" (g "process_name") "' on " host " queried high-entropy domain '" + (g "query_name") "' (score " (g "score") ")")) + ((string=? rule "impossible_travel") + (str "user '" (g "username") "' authenticated from " (g "country_a") " (" (g "ip_a") + ") and " (g "country_b") " (" (g "ip_b") ") within " (g "gap_minutes") " min")) + ((string=? rule "frequency_spike") + (str (g "event_type") " on " host ": " (g "count") " events in " (hour-bucket ts) + " (avg: " (fmt1 (g "average")) ")")) + ((string=? rule "severity_cluster") + (str (g "count") " critical/high events in 5min on " host " at " (format-ts ts))) + ((string=? rule "off_hours") + (str "Off-hours " (hash-get a "severity") " event: " (g "event_type") " on " host " at " (format-ts ts))) + ((string=? rule "kill_chain") + (str "Kill chain pattern on " host ": " (length (g "phases")) " phases in 1h at " (format-ts ts))) + (else "")))) + +;; ── detection assembly (mirrors secmon store.run_detections / detect_anomalies) ─ +(def db-detectors + (list (cons "brute_force" detect-brute-force) + (cons "credential_stuffing" detect-credential-stuffing) + (cons "dns_tunnel" detect-dns-tunnel) + (cons "suspicious_cron" detect-suspicious-cron) + (cons "recon_port_scan" detect-recon-port-scan) + (cons "data_exfil" detect-data-exfil) + (cons "priv_escalation_chain" detect-priv-escalation-chain) + (cons "persistence_after_access" detect-persistence-after-access) + (cons "log_cover" detect-log-cover) + (cons "lateral_after_shell" detect-lateral-after-shell) + (cons "impossible_travel" detect-impossible-travel))) + +(def (run-one-detection db rule filter) + (let ((p (assoc rule db-detectors))) + (cond (p ((cdr p) db filter)) + ((string=? rule "suspicious_cmdline") + (detect-lolbin-cmdline (query-events db (filter-set filter "limit" 1000000)))) + ((string=? rule "dga_domain") + (detect-dga-domain (query-events db (filter-set filter "limit" 1000000)))) + (else '())))) + +(def (sort-by-ts xs) + (list-sort (lambda (a b) (< (hash-get a "timestamp_ms") (hash-get b "timestamp_ms"))) xs)) + +;; rule: #f (all) or a name; unknown name errors exactly like secmon run_detections. +(def (run-all-detections db rule filter) + (let ((names (if rule + (if (rule-known? rule) (list rule) + (die "Error: unknown rule: '" rule "'. Use --list-rules to see available rules.")) + (rule-names)))) + (let ((all (append-map (lambda (r) (run-one-detection db r filter)) names))) + (for-each (lambda (a) (hash-put! a "attack" (rule-attack (hash-get a "rule")))) all) + (sort-by-ts all)))) + +(def (run-anomalies db filter) + (let ((all (run-anomaly-detections db filter))) + (for-each (lambda (a) (hash-put! a "attack" (anomaly-rule-attack (hash-get a "rule")))) all) + all)) + +;; YAML rules: --rules <dir|file>; run each whose name matches --rule (or all). +(def (yaml-rules-from-args args) + (let ((path (parse-flag-value args "--rules"))) + (if path + (try (load-yaml-rules path) + (catch (e) (die "Error: " e))) + '()))) + +(def (run-yaml-into db rules rule-name filter dets) + (let ((acc dets)) + (for-each + (lambda (r) + (when (or (not rule-name) (string=? (yaml-rule-name r) rule-name)) + (set! acc (append acc (run-yaml-rule db r filter))))) + rules) + acc)) + +(def (anomaly->json a) + (obj "rule" (hash-get a "rule") + "description" (describe-anomaly a) + "severity" (hash-get a "severity") + "@timestamp" (format-ts-iso (hash-get a "timestamp_ms")) + "host" (hash-get a "host") + "attack" (hash-get a "attack") + "details" (hash-get a "details"))) + +;; ── triage filter plumbing (exclude known-noise event ids) ──────────────────── +(def (apply-triage db filter no-triage) ;; → (values filter' triaged-count) + (if no-triage + (values filter 0) + (let ((ids (compute-triaged-ids db filter))) + (values (filter-set filter "exclude_event_ids" ids) (length ids))))) + +;; ── alert sinks (stdout / file / webhook / syslog) ──────────────────────────── +(def (shell-quote s) (string-append "'" (string-join (string-split s #\') "'\\''") "'")) + +(def (sink-dispatch sink line) + (case (car sink) + ((stdout) (println line) (ok #t)) + ((file) + (let* ((path (cadr sink)) + (old (if (file-exists? path) (read-file-string path) ""))) + (try (begin (call-with-output-file path + (lambda (o) (display old o) (display line o) (newline o)) 'replace) + (ok #t)) + (catch (e) (err (str "write " path ": " e)))))) + ((webhook syslog) + (let ((tmp (str "/tmp/jsecmon-alert-" (now-ms) "-" (random 100000)))) + (call-with-output-file tmp (lambda (o) (display line o)) 'replace) + (let ((rc (system + (if (eq? (car sink) 'webhook) + (str "curl -sS --max-time 10 -X POST " + "-H 'Content-Type: application/json' --data-binary @" + tmp " " (shell-quote (cadr sink)) " >/dev/null") + (str "logger -t " (shell-quote (cadr sink)) + " -p auth.warning -f " tmp))))) + (when (file-exists? tmp) (delete-file tmp)) + (if (= rc 0) (ok #t) (err (str (car sink) " exited " rc)))))) + (else (err "unknown sink")))) + +(def (dispatch-anomalies sinks anomalies) + (for-each + (lambda (a) + (let ((line (->json (anomaly->json a)))) + (for-each + (lambda (s) + (let ((r (sink-dispatch s line))) + (when (err? r) (eprintln "alert sink " (car s) " failed: " (unwrap-err r))))) + sinks))) + anomalies)) + +;; ── summary ─────────────────────────────────────────────────────────────────── +(def (cmd-summary db) + (let* ((s (store-summary db)) + (total (hash-get s "total")) (earliest (hash-get s "earliest_ms")) + (latest (hash-get s "latest_ms"))) + (println "Database Summary") + (println "================") + (println "Total events: " total) + (println "Hosts: " (length (hash-get s "hosts"))) + (when (and earliest latest) + (println "Time range: " (format-ts earliest) " to " (format-ts latest)) + (let ((hours (/ (exact->inexact (- latest earliest)) 3600000.0))) + (if (> hours 24.0) + (println "Duration: " (fmt1 (/ hours 24.0)) " days") + (println "Duration: " (fmt1 hours) " hours")))) + (println "") + (println "By Severity:") + (for-each (lambda (p) + (let ((pct (if (> total 0) (* (/ (exact->inexact (cdr p)) total) 100.0) 0.0))) + (println " " (pad-right (car p) 10) " " (pad-left (cdr p) 8) " (" (fmt1 pct) "%)"))) + (hash-get s "by_severity")) + (println "") + (println "By Event Type:") + (for-each (lambda (p) (println " " (pad-right (car p) 30) " " (pad-left (cdr p) 8))) + (hash-get s "by_type")) + (when (> (length (hash-get s "hosts")) 1) + (println "") + (println "By Host:") + (for-each (lambda (p) (println " " (pad-right (car p) 30) " " (pad-left (cdr p) 8))) + (hash-get s "by_host"))))) + +;; ── query ───────────────────────────────────────────────────────────────────── +(def (event->json ev) + (obj "id" (hash-get ev "id") "seq" (hash-get ev "seq") + "host" (hash-get ev "host") "source" (hash-get ev "source") + "@timestamp" (format-ts-iso (hash-get ev "timestamp_ms")) + "event_type" (hash-get ev "event_type") "severity" (hash-get ev "severity") + "pid" (jn (hash-get ev "pid")) "process_name" (jn (hash-get ev "process_name")) + "summary" (hash-get ev "summary") "data" (hash-get ev "data"))) + +(def (print-event-row ev) + (println "[" (format-ts (hash-get ev "timestamp_ms")) "] " (sev-marker (hash-get ev "severity")) + " seq=" (hash-get ev "seq") " host=" (hash-get ev "host")) + (println " " (pad-right (hash-get ev "event_type") 20) " " (hash-get ev "summary")) + (let ((pid (hash-get ev "pid")) (pn (hash-get ev "process_name"))) + (when pid + (if pn (println " pid=" pid " proc=" pn) (println " pid=" pid)))) + (println "")) + +(def (cmd-query db args) + (let ((events (query-events db (parse-filter args)))) + (if (is-json-format args) + (for-each (lambda (e) (print-json (event->json e))) events) + (if (null? events) (println "No events found.") + (begin (println (length events) " events found:") (println "") + (for-each print-event-row events)))))) + +;; ── anomalies ───────────────────────────────────────────────────────────────── +(def (cmd-anomalies db args) + (let ((json (is-json-format args)) + (no-triage (has-flag args "--no-triage")) + (sinks (let ((r (parse-alert-sinks args))) + (if (ok? r) (unwrap r) (die "Error: " (unwrap-err r)))))) + (let-values (((filter triaged) (apply-triage db (parse-filter args) no-triage))) + (let ((anoms (run-anomalies db filter))) + (cond + ((not (null? sinks)) + (dispatch-anomalies sinks anoms) + (eprintln "dispatched " (length anoms) " anomalies to " (length sinks) " sink(s)")) + (json (for-each (lambda (a) (print-json (anomaly->json a))) anoms)) + (else + (when (> triaged 0) (eprintln "(triage filtered out " triaged " known-benign events)")) + (if (null? anoms) (println "No anomalies detected.") + (begin (println (length anoms) " anomalies detected:") (println "") + (for-each + (lambda (a) + (println "[" (format-ts (hash-get a "timestamp_ms")) "] " + (sev-marker (hash-get a "severity")) " [" (hash-get a "rule") "] " + (describe-anomaly a) " (" (hash-get a "host") ")")) + anoms))))))))) + +;; ── first-seen ──────────────────────────────────────────────────────────────── +(def (cmd-first-seen db args) + (let ((bstr (or (parse-flag-value args "--baseline") + (die "Error: --baseline <datetime> is required")))) + (let ((bms (datetime-or-die bstr "--baseline")) + (host (parse-flag-value args "--host")) + (json (is-json-format args))) + (let ((entries (first-seen-after db bms host))) + (if json + (for-each (lambda (e) + (print-json (obj "value" (hash-get e "value") + "category" (hash-get e "category") + "first_seen" (format-ts-iso (hash-get e "first_seen_ms")) + "event_type" (hash-get e "event_type") + "host" (hash-get e "host")))) + entries) + (if (null? entries) + (println "No new items found after baseline " (format-ts bms) ".") + (begin + (println "First seen after " (format-ts bms) " (" (length entries) " items):") + (println "") + (let ((cur "")) + (for-each + (lambda (e) + (let ((cat (hash-get e "category"))) + (unless (string=? cat cur) (set! cur cat) (println " [" cat "]")) + (println " " (hash-get e "value") " " (hash-get e "event_type") + " (host: " (hash-get e "host") ", first: " + (format-ts (hash-get e "first_seen_ms")) ")"))) + entries))))))))) + +;; ── timeline ────────────────────────────────────────────────────────────────── +(def (cmd-timeline db args) + (let ((host (or (parse-flag-value args "--host") + (die "Error: --host <hostname> is required")))) + (let* ((filter (filter-set (parse-filter args) "host" host)) + (json (is-json-format args)) + (events (reverse (query-events db filter)))) ;; chronological + (if json + (for-each (lambda (e) + (print-json (obj "@timestamp" (format-ts-iso (hash-get e "timestamp_ms")) + "event_type" (hash-get e "event_type") + "severity" (hash-get e "severity") + "summary" (hash-get e "summary") + "pid" (jn (hash-get e "pid")) + "process_name" (jn (hash-get e "process_name"))))) + events) + (if (null? events) (println "No events found.") + (begin (println "Timeline (" (length events) " events):") (println "") + (for-each + (lambda (e) + (println (format-ts (hash-get e "timestamp_ms")) " " + (sev-marker (hash-get e "severity")) " " + (pad-right (hash-get e "event_type") 20) " " (hash-get e "summary"))) + events))))))) + +;; ── retention ───────────────────────────────────────────────────────────────── +(def (cmd-retention db args) + (let ((bstr (or (parse-flag-value args "--delete-before") + (die "Error: --delete-before <datetime> is required")))) + (let ((bms (datetime-or-die bstr "--delete-before"))) + (let ((n (delete-before db bms))) + (println "Deleted " n " events before " (format-ts bms)) + (when (has-flag args "--vacuum") + (display "Running VACUUM...") + (try (begin (store-vacuum db) (println " done.")) + (catch (e) (eprintln " error: " e)))))))) + +;; ── hosts ───────────────────────────────────────────────────────────────────── +(def (cmd-hosts db) + (let ((hosts (list-hosts db))) + (if (null? hosts) (println "No hosts found in database.") + (begin + (println "Known hosts:") (println "") + (println (pad-right "Source" 25) " " (pad-right "Hostname" 15) " " + (pad-left "Last Seq" 10) " Last Seen") + (println (make-string 70 #\-)) + (for-each + (lambda (h) + (println (pad-right (hash-get h "source") 25) " " + (pad-right (or (hash-get h "hostname") "-") 15) " " + (pad-left (hash-get h "last_seq") 10) " " + (format-ts (hash-get h "last_seen_ms")))) + hosts))))) + +;; ── ioc ─────────────────────────────────────────────────────────────────────── +(def (ioc-type->display sym) ;; Rust {:?} on IocType (capitalised) + (case sym ((ip) "Ip") ((domain) "Domain") ((hash) "Hash") ((process) "Process") + (else (symbol->string sym)))) + +(def (load-ioc-file path forced) + (let ((iocs (parse-ioc-text (read-file-string path)))) + (if forced (map (lambda (p) (list (car p) forced)) iocs) iocs))) + +(def (cmd-ioc db args) + (let ((list-path (or (parse-flag-value args "--list")