Port secmon event_json.rs: flat-JSON event data serializer
ober
96c533f03e310df8f1d70013e72826025543bd9c
--- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCHEME ?= $(JERBOA)/.chez/bin/scheme BUILD ?= build/rust TYPED := $(wildcard typed/*.ss) -.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-dns-check monitor-manager-check checks clean +.PHONY: rust test ffi-demo kernels-check triage-check triage-store-check analytics-check detect-check storage-check entity-check threats-check geoip-check sigma-check yaml-rules-check buffer-check dns-sniffer-check suspicious-check netconn-check kernmod-check selinux-check container-check dns-servers-check sensitive-path-check dtrace-parse-check proc-linux-check freebsd-parse-check event-meta-check config-check event-danger-check persistence-check file-change-check webshell-check platform-mounts-check analyze-cli-check collector-cli-check event-summary-check ioc-check frame-check correlate-check revshell-check cron-check logtamper-check detection-rules-check ipaddr-check auth-check lolbin-check dga-check calendar-check monitor-process-check monitor-network-check monitor-files-check monitor-dns-check monitor-manager-check event-json-check checks clean # Combined libdir path so sibling libraries `(jsecmon ...)` resolve to ./jsecmon # (a second --libdirs would replace, not append, the jerboa one). LIBDIRS := "$(JERBOA)/lib:$(CURDIR)" @@ -373,6 +373,12 @@ monitor-dns-check: monitor-manager-check: $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_manager_check.ss +# Flat-JSON event serializer (secmon event_json.rs): drives real monitor events +# through event-data-json and parses them back to assert each variant's exact +# data shape (null/dropped fields, nested parent, Debug change_type). No dylib. +event-json-check: + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_json_check.ss + # Everything that runs through the Jerboa side of the bridge, one shot. checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/triage_check.ss @@ -426,6 +432,7 @@ checks: kernels-check $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_files_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_dns_check.ss $(SCHEME) --libdirs $(LIBDIRS) --script examples/monitor_manager_check.ss + $(SCHEME) --libdirs $(LIBDIRS) --script examples/event_json_check.ss clean: rm -rf $(BUILD) new file mode 100644 --- /dev/null +++ b/examples/event_json_check.ss @@ -0,0 +1,192 @@ +;;; Behaviour check for the flat-JSON event serializer (secmon event_json.rs). +;;; +;;; Drives REAL monitor events (process/network/files/dns + agent_start) through +;;; `event-data-json`, parses the result back with `string->json-object`, and +;;; asserts the recovered fields — so we test the exact data each running monitor +;;; would emit, not hand-built hashes. Object key order is non-deterministic, so +;;; nothing string-compares whole JSON; null fields are checked on the wire form +;;; ("field":null, the shape a serde/Option::None consumer needs) and dropped +;;; fields by their absence. Every per-variant field set is pinned to the Rust. +;;; +;;; Run from the repo root with the repo on the libdir path: +;;; scheme --libdirs $JERBOA/lib --libdirs . --script examples/event_json_check.ss + +(import (jerboa prelude) + (jsecmon monitor-process) + (jsecmon monitor-network) + (jsecmon monitor-files) + (jsecmon monitor-dns) + (jsecmon event-json)) + +(def fails 0) +(def (check label got want) + (let ((ok (equal? got want))) + (unless ok (set! fails (+ fails 1))) + (displayln (if ok " ok " " FAIL ") label " => " got + (if ok "" (str " (want " want ")"))))) +(def (j ev) (event-data-json ev)) ;; the wire string +(def (p ev) (string->json-object (event-data-json ev))) ;; parsed back +(def (has? s sub) (and (string-contains s sub) #t)) +(def (jnull? ev field) (has? (j ev) (str "\"" field "\":null"))) + +;; --- generate real events ----------------------------------------------------- + +;; process: init (parent null, cwd null, empty environ), gitea (parent=init, +;; cwd + environ present), bash spawned by gitea -> suspicious_exec, then exit. +(def *pids* '()) +(def *procs* (make-hash-table)) +(hash-put! *procs* 1 (make-proc-info 1 0 0 "/sbin/init" "systemd" '("/sbin/init") #f '())) +(hash-put! *procs* 100 (make-proc-info 100 1 1000 "/usr/bin/gitea" "gitea" '("/usr/bin/gitea" "web") "/srv" '("PATH=/bin"))) +(hash-put! *procs* 300 (make-proc-info 300 100 1000 "/bin/bash" "bash" '("/bin/bash") #f '())) +(def pp (make-mon-provider (lambda () *pids*) (lambda (x) (hash-get *procs* x)) "host")) +(def pst (make-monitor pp)) +(set! *pids* '(1 100)) (def evP1 (scan-processes pst pp 1000)) +(set! *pids* '(1 100 300)) (def evP2 (scan-processes pst pp 1001)) +(set! *pids* '(1 300)) (def evP3 (scan-processes pst pp 1002)) +(def ev-ps-init (car evP1)) +(def ev-ps-gitea (cadr evP1)) +(def ev-susp-exec (car evP2)) +(def ev-exit (car evP3)) + +;; network: a listener, a benign conn (pid null), a reverse-shell-port conn. +(def l-ssh (make-conn-info "tcp" "0.0.0.0" 22 "0.0.0.0" 0 "LISTEN" #f "sshd")) +(def c-ok (make-conn-info "tcp" "10.0.0.2" 50000 "93.184.216.34" 443 "ESTABLISHED" #f "curl")) +(def c-4444 (make-conn-info "tcp" "10.0.0.2" 50001 "10.0.0.5" 4444 "ESTABLISHED" #f #f)) +(def np (make-net-provider (lambda () (list c-ok c-4444)) (lambda () (list l-ssh)) "host")) +(def evN (scan-connections (make-network-monitor np) np 1000)) +(def ev-listen (car evN)) +(def ev-netconn (cadr evN)) +(def ev-suspconn (caddr evN)) + +;; files: passwd content change (critical -> suspicious), hosts mtime (benign). +(def *fs* (make-hash-table)) +(def (fs-get x) (or (hash-get *fs* x) (make-file-state #f 0 0 0 0 #f))) +(hash-put! *fs* "/etc/passwd" (make-file-state "p1" #o644 0 0 100 #t)) +(hash-put! *fs* "/etc/hosts" (make-file-state "h1" #o644 0 0 100 #t)) +(def fp (make-file-provider fs-get (lambda (x) #f) "host")) +(def fst (make-file-monitor fp 'linux)) +(baseline-files fst fp '("/etc/passwd" "/etc/hosts")) +(hash-put! *fs* "/etc/passwd" (make-file-state "p2" #o644 0 0 100 #t)) +(hash-put! *fs* "/etc/hosts" (make-file-state "h1" #o644 0 0 200 #t)) +(def evF (scan-files fst fp 1001)) +(def ev-suspfile (car evF)) +(def ev-filechg (cadr evF)) + +;; dns: a UDP query to a resolver on :53. +(def q-udp (make-conn-info "udp" "10.0.0.2" 40000 "8.8.8.8" 53 "ESTABLISHED" 1234 "curl")) +(def dnsp (make-net-provider (lambda () (list q-udp)) (lambda () '()) "host")) +(def ev-dns (car (scan-dns-connections (make-dns-monitor '("8.8.8.8") "host") dnsp 1000))) + +;; agent_start. +(def ev-agent (agent-start-event "host" "0.1.0" 999)) + +;; --- assertions --------------------------------------------------------------- + +(displayln "process_start (init): scalars, empty environ array, null cwd/parent:") +(let ((pi (p ev-ps-init))) + (check "pid" (hash-get pi "pid") 1) + (check "ppid" (hash-get pi "ppid") 0) + (check "uid" (hash-get pi "uid") 0) + (check "exe" (hash-get pi "exe") "/sbin/init") + (check "name" (hash-get pi "name") "systemd") + (check "cmdline" (hash-get pi "cmdline") '("/sbin/init")) + (check "environ []" (hash-get pi "environ") '())) +(check "cwd null" (jnull? ev-ps-init "cwd") #t) +(check "parent null" (jnull? ev-ps-init "parent") #t) + +(displayln "process_start (gitea): present cwd, environ array, nested parent:") +(let* ((pg (p ev-ps-gitea)) (par (hash-get pg "parent"))) + (check "cwd present" (hash-get pg "cwd") "/srv") + (check "environ arr" (hash-get pg "environ") '("PATH=/bin")) + (check "cmdline 2" (hash-get pg "cmdline") '("/usr/bin/gitea" "web")) + (check "parent.pid" (hash-get par "pid") 1) + (check "parent.name" (hash-get par "name") "systemd") + (check "parent.exe" (hash-get par "exe") "/sbin/init")) + +(displayln "suspicious_exec: reason + nested parent; drops cwd:") +(let* ((se (p ev-susp-exec)) (par (hash-get se "parent"))) + (check "reason str" (string? (hash-get se "reason")) #t) + (check "pid" (hash-get se "pid") 300) + (check "exe" (hash-get se "exe") "/bin/bash") + (check "parent gitea" (hash-get par "name") "gitea")) +(check "drops cwd" (has? (j ev-susp-exec) "\"cwd\"") #f) + +(displayln "process_exit: null exit_code:") +(check "pid" (hash-get (p ev-exit) "pid") 100) +(check "exit_code null" (jnull? ev-exit "exit_code") #t) + +(displayln "agent_start: renamed host/version keys:") +(let ((ag (p ev-agent))) + (check "agent_hostname" (hash-get ag "agent_hostname") "host") + (check "agent_version" (hash-get ag "agent_version") "0.1.0")) + +(displayln "network_connection: full tuple, null pid, present process_name:") +(let ((nc (p ev-netconn))) + (check "local_addr" (hash-get nc "local_addr") "10.0.0.2") + (check "local_port" (hash-get nc "local_port") 50000) + (check "remote_addr" (hash-get nc "remote_addr") "93.184.216.34") + (check "remote_port" (hash-get nc "remote_port") 443) + (check "protocol" (hash-get nc "protocol") "tcp") + (check "state" (hash-get nc "state") "ESTABLISHED") + (check "process_name" (hash-get nc "process_name") "curl")) +(check "pid null" (jnull? ev-netconn "pid") #t) + +(displayln "listening_port: local side only (drops remote + state):") +(let ((lp (p ev-listen))) + (check "local_addr" (hash-get lp "local_addr") "0.0.0.0") + (check "local_port" (hash-get lp "local_port") 22) + (check "protocol" (hash-get lp "protocol") "tcp") + (check "process_name" (hash-get lp "process_name") "sshd")) +(check "pid null" (jnull? ev-listen "pid") #t) +(check "drops remote" (has? (j ev-listen) "remote") #f) +(check "drops state" (has? (j ev-listen) "\"state\"") #f) + +(displayln "suspicious_connection: reason; drops state + process_name:") +(let ((sc (p ev-suspconn))) + (check "reason 4444" (has? (hash-get sc "reason") "4444") #t) + (check "remote_port" (hash-get sc "remote_port") 4444) + (check "protocol" (hash-get sc "protocol") "tcp")) +(check "pid null" (jnull? ev-suspconn "pid") #t) +(check "drops state" (has? (j ev-suspconn) "\"state\"") #f) +(check "drops process_name" (has? (j ev-suspconn) "process_name") #f) + +(displayln "file_changed: full set, Debug change_type, 0 uid/gid stay numbers:") +(let ((fc (p ev-filechg))) + (check "path" (hash-get fc "path") "/etc/hosts") + (check "change_type" (hash-get fc "change_type") "Modified") + (check "old_hash" (hash-get fc "old_hash") "h1") + (check "new_hash" (hash-get fc "new_hash") "h1") + (check "old_mode" (hash-get fc "old_mode") #o644) + (check "new_mode" (hash-get fc "new_mode") #o644) + (check "uid 0" (hash-get fc "uid") 0) + (check "gid 0" (hash-get fc "gid") 0)) + +(displayln "suspicious_file_change: reason + hashes only (drops modes/uid/gid):") +(let ((sf (p ev-suspfile))) + (check "reason Critical" (has? (hash-get sf "reason") "Critical") #t) + (check "path" (hash-get sf "path") "/etc/passwd") + (check "change_type" (hash-get sf "change_type") "Modified") + (check "old_hash" (hash-get sf "old_hash") "p1") + (check "new_hash" (hash-get sf "new_hash") "p2")) +(check "drops modes" (has? (j ev-suspfile) "old_mode") #f) +(check "drops uid" (has? (j ev-suspfile) "\"uid\"") #f) + +(displayln "dns_query: <unknown> name, protocol-typed, empty response_addrs:") +(let ((dq (p ev-dns))) + (check "query_name" (hash-get dq "query_name") "<unknown>") + (check "query_type" (hash-get dq "query_type") "UDP") + (check "server_addr" (hash-get dq "server_addr") "8.8.8.8") + (check "response_addrs []" (hash-get dq "response_addrs") '()) + (check "pid" (hash-get dq "pid") 1234) + (check "process_name" (hash-get dq "process_name") "curl")) + +(displayln "change-type-debug spells out the Rust Debug variants:") +(check "permission" (change-type-debug 'permission-changed) "PermissionChanged") +(check "owner" (change-type-debug 'owner-changed) "OwnerChanged") +(check "created" (change-type-debug 'created) "Created") +(check "deleted" (change-type-debug 'deleted) "Deleted") + +(newline) +(if (= fails 0) + (displayln "OK: every event variant serializes to its exact secmon JSON data shape.") + (begin (displayln fails " FAILURES") (exit 1))) new file mode 100644 --- /dev/null +++ b/jsecmon/event-json.ss @@ -0,0 +1,157 @@ +#!chezscheme +;;; jsecmon flat-JSON event serializer (secmon src/event_json.rs), untyped. +;;; +;;; Port of `get_event_json_data`: turn an event into the flat JSON `data` body +;;; stored in the events table (the column the done event-summary.ss reads). In +;;; secmon this maps the rich EventType enum to (category, severity, json). In +;;; jsecmon the monitors already emit row hashes carrying the category ("type"), +;;; the severity, and the raw proc-info / conn-info / file-change objects; this +;;; module is the deferred other half — building each variant's JSON `data`. +;;; +;;; Faithfulness points the Rust pins: +;;; * field names and per-variant field SETS are exactly secmon's json!{...} +;;; (e.g. suspicious_exec drops cwd; suspicious_connection drops state + +;;; process_name; listening_port keeps only the local side). +;;; * Rust Option::None serializes as JSON null — so an absent pid / cwd / +;;; process_name / hash / mode / exit_code becomes null (the (if #f #f) void +;;; that json-object->string renders as null), never false or omitted. +;;; * change_type is Rust's Debug form ("Modified", "PermissionChanged", …). +;;; * a parent process becomes a nested {pid,name,exe} object, or null. +;;; * dns_query always carries response_addrs (an empty array in the polling +;;; fallback, which can't see answers). +;;; +;;; JSON object key ORDER is not significant (the data is jsecmon's own, +;;; consumed by parsing it back), so this need not match serde's byte layout. +;;; Verified by examples/event_json_check.ss, which drives real monitor events +;;; through the serializer and parses the result back. + +(library (jsecmon event-json) + (export event-data-json event-data-hash change-type-debug json-null) + (import (except (chezscheme) + make-hash-table hash-table? + sort sort! + printf fprintf + path-extension path-absolute? + with-input-from-string with-output-to-string + iota 1+ 1- + partition + make-date make-time) + (except (jerboa prelude) meta atom?) + (jsecmon monitor-process) + (jsecmon monitor-network) + (jsecmon monitor-files)) + + ;; the JSON null sentinel: json-object->string renders the unspecified value + ;; as `null`. jn maps an absent field (#f == Rust None) to it. + (def json-null (if #f #f)) + (def (jn v) (if (eq? v #f) json-null v)) + + (def (mk-hash pairs) + (let ((h (make-hash-table))) + (for-each (lambda (p) (hash-put! h (car p) (cdr p))) pairs) + h)) + + ;; Rust #[derive(Debug)] form of FileChangeType. + (def (change-type-debug sym) + (case sym + ((created) "Created") + ((deleted) "Deleted") + ((modified) "Modified") + ((permission-changed) "PermissionChanged") + ((owner-changed) "OwnerChanged") + (else (symbol->string sym)))) + + ;; a parent proc-info -> nested {pid,name,exe}, or null. + (def (parent-json p) + (if p + (mk-hash (list (cons "pid" (proc-info-pid p)) + (cons "name" (proc-info-name p)) + (cons "exe" (proc-info-exe p)))) + json-null)) + + ;; build the flat data hash for one event row (dispatch on its "type"). + (def (event-data-hash ev) + (let ((type (hash-get ev "type"))) + (cond + ((string=? type "process_start") + (let ((p (hash-get ev "process"))) + (mk-hash (list (cons "pid" (proc-info-pid p)) + (cons "ppid" (proc-info-ppid p)) + (cons "uid" (proc-info-uid p)) + (cons "exe" (proc-info-exe p)) + (cons "name" (proc-info-name p)) + (cons "cmdline" (proc-info-cmdline p)) + (cons "cwd" (jn (proc-info-cwd p))) + (cons "environ" (proc-info-environ p)) + (cons "parent" (parent-json (hash-get ev "parent"))))))) + ((string=? type "suspicious_exec") + (let ((p (hash-get ev "process"))) + (mk-hash (list (cons "reason" (hash-get ev "reason")) + (cons "pid" (proc-info-pid p)) + (cons "ppid" (proc-info-ppid p)) + (cons "uid" (proc-info-uid p)) + (cons "exe" (proc-info-exe p)) + (cons "cmdline" (proc-info-cmdline p)) + (cons "environ" (proc-info-environ p)) + (cons "parent" (parent-json (hash-get ev "parent"))))))) + ((string=? type "process_exit") + (mk-hash (list (cons "pid" (hash-get ev "pid")) + (cons "exit_code" (jn (hash-get ev "exit_code")))))) + ((string=? type "agent_start") + (mk-hash (list (cons "agent_hostname" (hash-get ev "host")) + (cons "agent_version" (hash-get ev "version"))))) + ((string=? type "network_connection") + (let ((c (hash-get ev "connection"))) + (mk-hash (list (cons "local_addr" (conn-info-local-addr c)) + (cons "local_port" (conn-info-local-port c)) + (cons "remote_addr" (conn-info-remote-addr c)) + (cons "remote_port" (conn-info-remote-port c)) + (cons "protocol" (conn-info-protocol c)) + (cons "state" (conn-info-state c)) + (cons "pid" (jn (conn-info-pid c))) + (cons "process_name" (jn (conn-info-process-name c))))))) + ((string=? type "listening_port") + (let ((c (hash-get ev "connection"))) + (mk-hash (list (cons "local_addr" (conn-info-local-addr c)) + (cons "local_port" (conn-info-local-port c)) + (cons "protocol" (conn-info-protocol c)) + (cons "pid" (jn (conn-info-pid c))) + (cons "process_name" (jn (conn-info-process-name c))))))) + ((string=? type "suspicious_connection") + (let ((c (hash-get ev "connection"))) + (mk-hash (list (cons "reason" (hash-get ev "reason")) + (cons "local_addr" (conn-info-local-addr c)) + (cons "local_port" (conn-info-local-port c)) + (cons "remote_addr" (conn-info-remote-addr c)) + (cons "remote_port" (conn-info-remote-port c)) + (cons "protocol" (conn-info-protocol c)) + (cons "pid" (jn (conn-info-pid c))))))) + ((string=? type "file_changed") + (let ((c (hash-get ev "change"))) + (mk-hash (list (cons "path" (file-change-path c)) + (cons "change_type" (change-type-debug (file-change-change-type c))) + (cons "old_hash" (jn (file-change-old-hash c))) + (cons "new_hash" (jn (file-change-new-hash c))) + (cons "old_mode" (jn (file-change-old-mode c))) + (cons "new_mode" (jn (file-change-new-mode c))) + (cons "uid" (jn (file-change-uid c))) + (cons "gid" (jn (file-change-gid c))))))) + ((string=? type "suspicious_file_change") + (let ((c (hash-get ev "change"))) + (mk-hash (list (cons "reason" (hash-get ev "reason")) + (cons "path" (file-change-path c)) + (cons "change_type" (change-type-debug (file-change-change-type c))) + (cons "old_hash" (jn (file-change-old-hash c))) + (cons "new_hash" (jn (file-change-new-hash c))))))) + ((string=? type "dns_query") + (mk-hash (list (cons "query_name" (hash-get ev "query_name")) + (cons "query_type" (hash-get ev "query_type")) + (cons "server_addr" (hash-get ev "server_addr")) + (cons "response_addrs" '()) + (cons "pid" (jn (hash-get ev "pid"))) + (cons "process_name" (jn (hash-get ev "pname")))))) + (else (make-hash-table))))) + + ;; the flat JSON `data` string for an event row. + (def (event-data-json ev) + (json-object->string (event-data-hash ev))))