Port monitor/auth log-line parsers to (jsecmon auth)
ober
3964dba1b9ddad431c145aeb605caba4a2b570be
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -286,6 +286,12 @@ detection-rules-check: ipaddr-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/ipaddr_check.ss +# auth.ss: the auth-log line parsers (sshd / sudo / su / pam / useradd / userdel +# / passwd) and extract_field. Faithful to auth.rs incl. its case-sensitive +# field extraction and the username quirks; expectations from a Rust oracle. +auth-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/auth_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -325,6 +331,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/logtamper_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/detection_rules_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/ipaddr_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/auth_check.ss clean: rm -rf $(BUILD) --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ make cron-check # cron: per-platform CRON/PERIODIC path tables + systemd/pe make logtamper-check # logtamper: system-log/history tables + classify-tamper (trunc/mtime) make detection-rules-check # DETECTION_RULES ATT&CK catalog: rule_attack + anomaly_rule_attack make ipaddr-check # lateral: faithful IpAddr parse (v4/v6) + is_internal_ip (RFC1918/fc00) +make auth-check # auth-log parsers: sshd/sudo/su/pam/useradd/userdel/passwd + extract_field make checks # every Jerboa-side check in one shot ``` @@ -145,4 +146,5 @@ then crypto orchestration, then I/O / async / FFI (monitors, server, storage). | `monitor/logtamper` log-tamper decision core | `jsecmon/logtamper.ss` | ✅ **untyped layer** — the pure pieces of the log-tampering monitor (the `fs::metadata` polling + size/mtime tracking map stay in the loop). Exposes the `SYSTEM_LOGS` / `HISTORY_FILES` constant tables and `TRUNCATION_THRESHOLD` (1000). `is-history-file` is any-`HISTORY_FILES`-**substring**. `classify-tamper old-size new-size old-mtime new-mtime path` reproduces `check_tampering`'s Ok-arm in push order: size dropped by **>** threshold → `history-cleared` (if a history file) else `truncated`; mtime went backwards and `> 0` → `timestamp-modified` (both can fire for one file). The `deleted` case is the `fs::metadata` Err arm (deferred I/O). logtamper.rs has no #[test], so `make logtamper-check` **is** the spec. | | `storage` detection-rule ATT&CK catalog (`DETECTION_RULES` / `rule_attack` / `anomaly_rule_attack`) | `jsecmon/detection-rules.ss` | ✅ **untyped layer** — the static `DETECTION_RULES` table (13 rows: name / description / severity / MITRE techniques) and its pure lookups, used to annotate findings with ATT&CK IDs and to list/validate rule names (the SQL detectors that fire these live in `(jsecmon threats)`). `rule-attack` finds the row by exact name and returns its `attack` list, else `()` (unwrap_or_default); `rule-names` / `rule-known?` mirror the listing/validation paths. `anomaly-rule-attack` is the separate match for the `detect_anomalies` **statistical** rules (not in the table): `kill_chain` → recon/lateral/exfil tactics, `off_hours` → Defense Evasion, others (frequency_spike, severity_cluster) → `()`. const has no #[test], so `make detection-rules-check` asserts every row + both lookups and **is** the spec. | | `monitor/lateral` IP parsing + internal-network test (`is_internal_ip`) | `jsecmon/ipaddr.ss` | ✅ **untyped layer** — `is_internal_ip` parses `ip_str.parse::<IpAddr>()` first and only classifies on success, so `parse-ipv4` / `parse-ipv6` faithfully reproduce **Rust std's `IpAddr` FromStr** boundary: IPv4 = exactly 4 octets, 1–3 digits, **no leading zeros**, ≤255; IPv6 = colon-separated 1–4-digit hex groups with at most one `::` (eliding ≥1 zero group) and an optional trailing embedded IPv4 (forbidden before `::`). `is-internal-ip` then mirrors lateral.rs: V4 `10/8` · `172.16/12` · `192.168/16` · `127/8`, V6 `fc00::/7` (`seg0 & 0xfe00 == 0xfc00`) or loopback. lateral.rs has no #[test]; expectations were generated by a **std-only Rust oracle** over ~70 inputs, so `make ipaddr-check` (88 cases) pins the port to real Rust and **is** the spec. | +| `monitor/auth` log-line parsers (`parse_auth_line` + 7 sub-parsers) | `jsecmon/auth.ss` | ✅ **untyped layer** — the dispatcher lowercases and routes on substrings (`sshd`→ssh, `sudo`, `su[`/`su:`, `authentication failure`→pam, `useradd`/`adduser`, `userdel`/`deluser`, `password changed`/`passwd`), each sub-parser pulling fields with `extract_field(line, start, end)`. Faithful corners: `extract_field` searches the **raw** line (case-sensitive) while dispatch/success use the lower-cased copy, so `Invalid user admin` → username `unknown`; the SSH "accepted" branch `?`-propagates a missing `for ` to **#f** (no fall-through); and the documented quirks where the username captures `TTY=pts/0` / `authentication failure; TTY=…` / `root)` are preserved. AuthEventType → `'ssh-key-auth 'login 'failed-login 'sudo-attempt 'su-attempt 'user-created 'user-deleted 'password-change`; None → `#f`. auth.rs has no #[test]; `make auth-check` (24 cases) derives every expectation from a **std-only Rust oracle** over the verbatim bodies and **is** the spec. | | monitors / server / ebpf / dtrace | — | ⏳ I/O+async+FFI, last | new file mode 100644 --- /dev/null +++ b/examples/auth_check.ss @@ -0,0 +1,113 @@ +;;; Parity check for (jsecmon auth) against secmon src/monitor/auth.rs. +;;; auth.rs has no #[test]; every expectation below was generated by a std-only +;;; Rust oracle compiling auth.rs's extract_field + parse_* bodies verbatim and +;;; dumping the parsed fields over a battery of dispatcher and direct inputs. +;;; +;;; scheme --libdirs "$JERBOA/lib:." --script examples/auth_check.ss + +(import (jerboa prelude) + (jsecmon auth)) + +(def fails 0) +(def (check name got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") name + (if ok "" (str " got " got " want " want))))) + +;; expected auth-event builder — same field order auth.rs/auth-event use +(def (ev type username tty remote success message) + (list (cons 'type type) (cons 'username username) (cons 'tty tty) + (cons 'remote-host remote) (cons 'success success) + (cons 'message message))) + +;; ── extract-field: first start, up to first following end (or eol), trimmed; +;; empty -> #f. Case-sensitive find on the raw line. ───────────────────────── +(displayln "extract-field:") +(check "for/from" (extract-field "Accepted for alice from 1.2.3.4 port 22" "for " " from") "alice") +(check "name=," (extract-field "name=frank, UID=1" "name=" ",") "frank") +(check "end missing -> to eol" + (extract-field "x COMMAND=/bin/ls" "COMMAND=" "\n") "/bin/ls") +(check "start missing -> #f" + (extract-field "no start token here" "for " " from") #f) +(check "empty field -> #f" + (extract-field "for from" "for " " from") #f) + +;; ── parse-auth-line dispatcher (Rust oracle ground truth) ───────────────────── +(displayln "parse-auth-line:") +(def l1 "host sshd[1]: Accepted publickey for alice from 10.0.0.5 port 4242 ssh2") +(check "ssh accepted publickey" + (parse-auth-line l1) + (ev 'ssh-key-auth "alice" #f "10.0.0.5" #t l1)) +(def l2 "host sshd[1]: Accepted password for bob from 192.168.1.9 port 22 ssh2") +(check "ssh accepted password" + (parse-auth-line l2) + (ev 'login "bob" #f "192.168.1.9" #t l2)) +(def l3 "host sshd[1]: Failed password for carol from 203.0.113.7 port 22 ssh2") +(check "ssh failed" + (parse-auth-line l3) + (ev 'failed-login "carol" #f "203.0.113.7" #f l3)) +(def l4 "host sshd[1]: Failed password for invalid user dave from 1.2.3.4 port 22 ssh2") +(check "ssh failed keeps 'invalid user dave' username" + (parse-auth-line l4) + (ev 'failed-login "invalid user dave" #f "1.2.3.4" #f l4)) +(check "ssh 'Invalid user' (capital) -> username unknown, msg 'Invalid user'" + (parse-auth-line "host sshd[1]: Invalid user admin from 1.2.3.4 port 22") + (ev 'failed-login "unknown" #f "1.2.3.4" #f "Invalid user")) +(check "ssh Accepted with no 'for ' -> #f (? propagates)" + (parse-auth-line "host sshd[1]: Accepted publickey from 10.0.0.1 port 22") #f) +(check "ssh non-auth line -> #f" + (parse-auth-line "host sshd[1]: Connection closed by 10.0.0.1") #f) + +(def s1 "host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/bin/ls") +(check "sudo success -> username quirk 'TTY=pts/0', tty pts/0, command msg" + (parse-auth-line s1) + (ev 'sudo-attempt "TTY=pts/0" "pts/0" #f #t "/bin/ls")) +(def s2 "host sudo: bob : authentication failure; TTY=pts/1 ; PWD=/ ; USER=root ; COMMAND=/bin/sh") +(check "sudo auth-failure -> not success, fixed msg" + (parse-auth-line s2) + (ev 'sudo-attempt "authentication failure; TTY=pts/1" "pts/1" #f #f "sudo authentication failed")) + +(def su1 "host su: (to root) alice on pts/0") +(check "su (to root) -> success, username quirk 'root)'" + (parse-auth-line su1) + (ev 'su-attempt "root)" #f #f #t su1)) +(def su2 "host su[123]: FAILED su for root by alice") +(check "su FAILED -> not success, username root" + (parse-auth-line su2) + (ev 'su-attempt "root" #f #f #f su2)) + +(def p1 "host login: pam_unix(login:auth): authentication failure; logname=root uid=0 user=eve rhost=1.2.3.4") +(check "pam failure -> failed-login, user=eve" + (parse-auth-line p1) + (ev 'failed-login "eve" #f #f #f p1)) + +(def ua "host useradd[1]: new user: name=frank, UID=1001, GID=1001") +(check "useradd -> user-created frank" + (parse-auth-line ua) + (ev 'user-created "frank" #f #f #t ua)) +(def ud "host userdel[1]: removed user name=grace, UID=1002") +(check "userdel -> user-deleted grace" + (parse-auth-line ud) + (ev 'user-deleted "grace" #f #f #t ud)) +(def pw "host passwd[1]: password changed for henry") +(check "passwd -> password-change henry" + (parse-auth-line pw) + (ev 'password-change "henry" #f #f #t pw)) + +(check "unrelated -> #f" + (parse-auth-line "random unrelated line") #f) + +;; ── direct sub-parser calls (pin corners hard to route) ─────────────────────── +(displayln "direct parsers:") +(check "parse-ssh-line Accepted w/o 'for ' -> #f" + (parse-ssh-line "host sshd[1]: Accepted publickey from 10.0.0.1 port 22") #f) +(def pd "pam_unix(sshd:auth): authentication failure; user=ivan )") +(check "parse-pam-failure user= up to space -> ivan" + (parse-pam-failure pd) + (ev 'failed-login "ivan" #f #f #f pd)) + +(newline) +(if (= fails 0) + (displayln "OK: auth matches secmon's monitor/auth.rs parsers.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/auth.ss @@ -0,0 +1,166 @@ +#!chezscheme +;;; jsecmon authentication-log parsers (secmon src/monitor/auth.rs), untyped. +;;; +;;; The pure string parsers of the auth monitor, with the file tailing, event +;;; emission, and dedup left to the monitor loop: +;;; extract-field : line start end -> field-string | #f +;;; parse-ssh-line : line -> auth-event | #f +;;; parse-sudo-line : line -> auth-event | #f +;;; parse-su-line : line -> auth-event | #f +;;; parse-pam-failure : line -> auth-event +;;; parse-user-add : line -> auth-event +;;; parse-user-del : line -> auth-event +;;; parse-passwd-change : line -> auth-event +;;; parse-auth-line : line -> auth-event | #f (dispatcher) +;;; +;;; An auth-event is the alist +;;; ((type . sym) (username . str) (tty . str|#f) (remote-host . str|#f) +;;; (success . bool) (message . str|#f)) +;;; mirroring AuthEventInfo; AuthEventType maps to the symbols 'ssh-key-auth +;;; 'login 'failed-login 'sudo-attempt 'su-attempt 'user-created 'user-deleted +;;; 'password-change. Rust Option None becomes #f. +;;; +;;; CRITICAL faithful corner: extract_field runs on the ORIGINAL line (its +;;; .find is case-sensitive) while the dispatcher and the sudo/su success tests +;;; run on a lower-cased copy. So "Invalid user admin from ..." dispatches on +;;; the lower-cased "invalid user" yet extract-field "invalid user " misses the +;;; capitalised text and yields "unknown". This split is reproduced exactly: +;;; extract-field is always handed `line`, the contains tests are handed `ll`. +;;; +;;; Other faithful quirks (all confirmed against a std-only Rust oracle): +;;; - parse_ssh_line "accepted" uses `?` on the username, so an Accepted line +;;; with no "for " (e.g. "Accepted publickey from H port P") yields #f, NOT +;;; a fall-through to the "failed"/"invalid user" branches. +;;; - extract-field "for " " from" on a "Failed password for invalid user dave +;;; from .." line keeps the literal username "invalid user dave". +;;; - sudo " : " " ;" can capture "TTY=pts/0" / "authentication failure; TTY=" +;;; when the real username is absent; su "to " " " can capture "root)". +;;; extract_field splices/searches by byte index in Rust; auth log lines are +;;; ASCII so the char-indexed string ops here are identical. +;;; +;;; auth.rs has no #[test]; examples/auth_check.ss derives every expectation +;;; from a Rust oracle over the auth.rs bodies and IS the spec for this port. + +(library (jsecmon auth) + (export extract-field + parse-ssh-line parse-sudo-line parse-su-line parse-pam-failure + parse-user-add parse-user-del parse-passwd-change + parse-auth-line) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?)) + + ;; extract_field(line, start, end): from the first `start`, take up to the + ;; first following `end` (or end-of-string), trim; empty -> #f. + (def (extract-field line start end) + (let ((si (string-contains line start))) + (if (not si) + #f + (let* ((start-idx (+ si (string-length start))) + (remaining (substring line start-idx (string-length line))) + (ei (string-contains remaining end)) + (end-idx (if ei ei (string-length remaining))) + (field (string-trim (substring remaining 0 end-idx)))) + (if (string-empty? field) #f field))))) + + ;; boolean substring test (string-contains returns an index or #f) + (def (has? s sub) (if (string-contains s sub) #t #f)) + + (def (auth-event type username tty remote success message) + (list (cons 'type type) + (cons 'username username) + (cons 'tty tty) + (cons 'remote-host remote) + (cons 'success success) + (cons 'message message))) + + (def (parse-ssh-line line) + (let ((ll (string-downcase line))) + (cond + ((has? ll "accepted") + (let ((username (extract-field line "for " " from"))) + (if (not username) + #f + (auth-event (if (has? ll "publickey") 'ssh-key-auth 'login) + username #f + (extract-field line "from " " port") + #t line)))) + ((has? ll "failed") + (let ((username (or (extract-field line "for " " from") + (extract-field line "user " " from") + "unknown"))) + (auth-event 'failed-login username #f + (extract-field line "from " " port") + #f line))) + ((has? ll "invalid user") + (let ((username (or (extract-field line "invalid user " " from") + "unknown"))) + (auth-event 'failed-login username #f + (extract-field line "from " " port") + #f "Invalid user"))) + (else #f)))) + + (def (parse-sudo-line line) + (let ((ll (string-downcase line)) + (username (extract-field line " : " " ;"))) + (if (not username) + #f + (let* ((success (and (not (has? ll "authentication failure")) + (not (has? ll "incorrect password")))) + (tty (extract-field line "TTY=" " ")) + (message (if success + (extract-field line "COMMAND=" "\n") + "sudo authentication failed"))) + (auth-event 'sudo-attempt username tty #f success message))))) + + (def (parse-su-line line) + (let* ((ll (string-downcase line)) + (success (or (has? ll "successful") (not (has? ll "failed")))) + (username (or (extract-field line "for " " by") + (extract-field line "to " " ") + "unknown"))) + (auth-event 'su-attempt username #f #f success line))) + + (def (parse-pam-failure line) + (let ((username (or (extract-field line "user=" " ") + (extract-field line "user=" ")") + "unknown"))) + (auth-event 'failed-login username #f #f #f line))) + + (def (parse-user-add line) + (let ((username (or (extract-field line "name=" ",") + (extract-field line "name=" " ") + "unknown"))) + (auth-event 'user-created username #f #f #t line))) + + (def (parse-user-del line) + (let ((username (or (extract-field line "name=" ",") + (extract-field line "name=" " ") + "unknown"))) + (auth-event 'user-deleted username #f #f #t line))) + + (def (parse-passwd-change line) + (let ((username (or (extract-field line "for " " ") + (extract-field line "user " " ") + "unknown"))) + (auth-event 'password-change username #f #f #t line))) + + (def (parse-auth-line line) + (let ((ll (string-downcase line))) + (cond + ((has? ll "sshd") (parse-ssh-line line)) + ((has? ll "sudo") (parse-sudo-line line)) + ((or (has? ll "su[") (has? ll "su:")) (parse-su-line line)) + ((has? ll "authentication failure") (parse-pam-failure line)) + ((or (has? ll "useradd") (has? ll "adduser")) (parse-user-add line)) + ((or (has? ll "userdel") (has? ll "deluser")) (parse-user-del line)) + ((or (has? ll "password changed") (has? ll "passwd")) (parse-passwd-change line)) + (else #f)))) +)