docs(std-perf): trip-bytes is workload-dependent + scanner benches
ober
030091e709bed9ee2d53b107457878fc09c2e726
new file mode 100644 --- /dev/null +++ b/benchmarks/bench-ac-alloc.ss @@ -0,0 +1,123 @@ +#!/usr/bin/env scheme-script +#!chezscheme +;;; bench-ac-alloc.ss — measure AC allocation per scan. +;;; +;;; chez-limits.md "Things to try next" #2 hypothesised that AC scanning +;;; allocates a "current state" box per file. The current code in +;;; lib/std/text/aho-corasick.ss tracks state as a loop fixnum (zero +;;; alloc in the no-match path), so the hypothesis appears obsolete. +;;; +;;; This bench measures the actual allocation rate to confirm: +;;; - Scan a fixed bytevector N times with M patterns +;;; - Compare ac-search-fold (no list cons) vs ac-search (list cons) +;;; - Report bytes allocated per byte scanned + +(or (guard (_ [#t #f]) (load-shared-object "libc.so.7") #t) + (guard (_ [#t #f]) (load-shared-object "libc.so.6") #t) + (guard (_ [#t #f]) (load-shared-object "libc.so") #t) + (guard (_ [#t #f]) (load-shared-object "libSystem.dylib") #t) + (guard (_ [#t #f]) (load-shared-object "libSystem.B.dylib") #t)) + +;; CRITICAL: optimize-level 3 is required for the AC inner loop to compile +;; to a tight zero-allocation loop. At the default level, the same code +;; allocates ~256 bytes per byte scanned (a 1000x penalty on throughput). +(optimize-level 3) +(compile-imported-libraries #t) + +(import (chezscheme) + (std text aho-corasick) + (std runtime gc)) + +;; ---- Test data ---- +;; 4 KB random-looking bytevector (no actual matches for the patterns) +;; plus a small bytevector with planted matches. + +(random-seed 17) +(define haystack-clean + (let ([bv (make-bytevector 65536 0)]) + (let loop ([i 0]) + (when (< i 65536) + (bytevector-u8-set! bv i (random 256)) + (loop (+ i 1)))) + bv)) + +(define haystack-matches + ;; Same size but contains the pattern "EVIL" ~200 times. + (let ([bv (make-bytevector 65536 0)] + [pat (string->utf8 "EVIL")]) + (let loop ([i 0]) + (when (< i 65536) + (bytevector-u8-set! bv i (random 256)) + (loop (+ i 1)))) + (let plant ([i 0]) + (when (< i 200) + (let ([off (* i 320)]) + (when (< (+ off 4) 65536) + (bytevector-copy! pat 0 bv off 4))) + (plant (+ i 1)))) + bv)) + +(define patterns + (let ([words '("EVIL" "MALWARE" "SHELL" "BACKDOOR" "TROJAN" + "RANSOM" "EXPLOIT" "PAYLOAD" "INJECT" "ROOTKIT")]) + (let loop ([ws words] [i 0] [acc '()]) + (if (null? ws) + (reverse acc) + (loop (cdr ws) (+ i 1) + (cons (cons i (string->utf8 (car ws))) acc)))))) + +(define aut (make-ac patterns)) + +;; ---- Bench harness ---- + +(define (bench name iters thunk) + (gc-collect-now 'major) + (let* ([gc-before (gc-stats)] + [t-start (real-time)]) + (let loop ([i 0]) + (when (< i iters) + (thunk) + (loop (+ i 1)))) + (let* ([t-end (real-time)] + [gc-after (gc-stats)] + [alloc (- (gc-stats-bytes-allocated gc-after) + (gc-stats-bytes-allocated gc-before))] + [gcs (- (gc-stats-collection-count gc-after) + (gc-stats-collection-count gc-before))]) + (printf " ~a~%" name) + (printf " wall: ~5d ms, alloc: ~9d B (~,2f B/byte-scanned), GCs: ~a~%" + (- t-end t-start) alloc + (/ alloc (* iters 65536.0)) + gcs)))) + +;; ---- Runs ---- + +(printf "=== AC allocation profile ===~%~%") + +(printf "Haystack: 64 KB, ~a patterns, 4000 iters per case~%~%" (length patterns)) + +(printf "[no matches in haystack]~%") +(bench "ac-search-fold (counts matches, returns int)" + 4000 + (lambda () + (ac-search-fold aut + (lambda (id off len acc) (fx+ acc 1)) + 0 + haystack-clean))) +(bench "ac-search (builds list of (id off len) triples)" + 4000 + (lambda () + (ac-search aut haystack-clean))) + +(printf "~%[~~200 matches in haystack]~%") +(bench "ac-search-fold" + 4000 + (lambda () + (ac-search-fold aut + (lambda (id off len acc) (fx+ acc 1)) + 0 + haystack-matches))) +(bench "ac-search" + 4000 + (lambda () + (ac-search aut haystack-matches))) new file mode 100644 --- /dev/null +++ b/benchmarks/bench-gc-tuning.ss @@ -0,0 +1,143 @@ +#!/usr/bin/env scheme-script +#!chezscheme +;;; bench-gc-tuning.ss — Grid-search Chez GC knobs for scanner workloads. +;;; +;;; The "many small short-lived allocations" claim in docs/chez-limits.md +;;; "Things to try next" is hypothetical. This bench actually measures +;;; across a grid of (collect-trip-bytes × collect-generation-radix) using +;;; a representative scanner-shaped synthetic load: +;;; +;;; - allocate a fresh bytevector per file (mimics get-bytevector-all) +;;; - touch its bytes to defeat dead-store elimination +;;; - discard (let it die in gen-0) +;;; +;;; Reports wall + GC count + collector CPU for each (trip, radix) tuple. + +;; ---- libc preamble ---- +(or (guard (_ [#t #f]) (load-shared-object "libc.so.7") #t) + (guard (_ [#t #f]) (load-shared-object "libc.so.6") #t) + (guard (_ [#t #f]) (load-shared-object "libc.so") #t) + (guard (_ [#t #f]) (load-shared-object "libSystem.dylib") #t) + (guard (_ [#t #f]) (load-shared-object "libSystem.B.dylib") #t)) + +(import (chezscheme) + (std runtime gc)) + +;; ---- Synthetic scanner workload ---- +;; +;; Roughly mimics what virus-scan does per file: allocate a bytevector +;; sized like a typical binary (median ~200 KB across /usr/bin), touch +;; the first/last byte (forces materialisation), let it die. Iters +;; chosen so each config takes ~150 ms on the baseline (good +;; signal-to-noise without being slow). + +(define iters 4000) +(define sizes + ;; Mix of small / medium / large to keep gen distribution realistic. + ;; Fixed seed via random-seed so all configurations see the same + ;; size sequence (apples-to-apples comparison). + (begin + (random-seed 42) + (let loop ([i 0] [acc '()]) + (if (= i iters) + (list->vector acc) + (loop (+ i 1) + (cons (+ 4096 (random 524288)) acc)))))) + +(define (workload) + (let loop ([i 0] [sum 0]) + (cond + [(= i iters) sum] + [else + (let* ([sz (vector-ref sizes i)] + [bv (make-bytevector sz 0)]) + (bytevector-u8-set! bv 0 (bitwise-and i 255)) + (bytevector-u8-set! bv (- sz 1) (bitwise-and i 255)) + (loop (+ i 1) + (+ sum (bytevector-u8-ref bv 0) (bytevector-u8-ref bv (- sz 1)))))]))) + +;; Same workload, partitioned across N OS threads. This is the +;; configuration where S_get_more_room contention shows up — and the +;; reason docs/chez-limits.md cares about GC tuning at all. +(define (parallel-workload n-threads) + (let* ([per-thread (quotient iters n-threads)] + [threads + (let loop ([i 0] [acc '()]) + (if (= i n-threads) + acc + (let ([start (* i per-thread)] + [end (if (= i (- n-threads 1)) + iters + (* (+ i 1) per-thread))]) + (loop (+ i 1) + (cons (fork-thread + (lambda () + (let inner ([j start] [sum 0]) + (cond + [(= j end) sum] + [else + (let* ([sz (vector-ref sizes j)] + [bv (make-bytevector sz 0)]) + (bytevector-u8-set! bv 0 (bitwise-and j 255)) + (bytevector-u8-set! bv (- sz 1) (bitwise-and j 255)) + (inner (+ j 1) + (+ sum + (bytevector-u8-ref bv 0) + (bytevector-u8-ref bv (- sz 1)))))])))) + acc)))))]) + (for-each thread-join threads))) + +;; ---- Bench harness ---- + +(define (bench label thunk trip radix) + (let ([saved-trip (gc-trip-bytes)] + [saved-radix (gc-generation-radix)]) + (gc-trip-bytes trip) + (gc-generation-radix radix) + (gc-collect-now 'major) + (let* ([gc-before (gc-stats)] + [t-start (real-time)] + [_ (thunk)] + [t-end (real-time)] + [gc-after (gc-stats)]) + (gc-trip-bytes saved-trip) + (gc-generation-radix saved-radix) + (printf " ~a trip=~8d radix=~2d ~6d ms ~4d GCs ~,3f s collector~%" + label trip radix + (- t-end t-start) + (- (gc-stats-collection-count gc-after) + (gc-stats-collection-count gc-before)) + (- (gc-stats-collector-cpu-seconds gc-after) + (gc-stats-collector-cpu-seconds gc-before)))))) + +;; ---- Grid ---- + +(define trips + '(8388608 ;; 8 MB (Chez default) + 16777216 ;; 16 MB + 33554432 ;; 32 MB + 67108864 ;; 64 MB + 134217728 ;; 128 MB + 268435456 ;; 256 MB + 536870912)) ;; 512 MB +(define radii '(4)) ;; radix had negligible effect on prior runs; fix at 4 + +(printf "=== GC tuning grid search ===~%") +(printf "Workload: ~a iters, bytevectors 4 KB-512 KB random sizes~%~%" iters) + +(define (do-sweep label thunk) + (printf "~%~a~%" label) + (printf " Baseline (Chez defaults):~%") + (bench "[1T]" thunk 8388608 4) + (newline) + (for-each + (lambda (trip) + (for-each + (lambda (radix) + (bench " " thunk trip radix)) + radii)) + trips)) + +(do-sweep "=== Single-threaded ===" workload) +(do-sweep "=== 8 threads ===" (lambda () (parallel-workload 8))) +(do-sweep "=== 14 threads ===" (lambda () (parallel-workload 14))) --- a/docs/std-perf.md +++ b/docs/std-perf.md @@ -291,6 +291,47 @@ that sets them silently surprises everyone else linked in. Use `with-gc-tuning` to scope changes to a measurable phase, or document the tuning as a deployment knob. +### Measured: `gc-trip-bytes` is workload-dependent + +The "right" value is the knee of YOUR allocation curve, not a fixed +recipe. Two measurements show how widely the optimum moves: + +**Synthetic bench** (`benchmarks/bench-gc-tuning.ss`) — 4000 iters +allocating 4 KB–512 KB bytevectors with no other work; 8/14 threads: + +| trip-bytes | wall (8T) | wall (14T) | +|------------|-----------|------------| +| 8 MB (default) | 1.0× | 1.0× | +| 64 MB | 0.58× | 0.62× | +| 256 MB | 0.65× | 0.70× | + +Sweet spot ≈ 64 MB — the curve is U-shaped because larger trips +defer GCs past the point where the surviving working set forces +expensive majors. + +**Real virus-scan workload** (`/usr/bin`, 874 files / 218 MB, j=8, +3 runs each, median): + +| trip-bytes | scan ms | peak RSS | +|------------|---------|----------| +| 8 MB (default) | 3856 | 5.14 GB | +| 32 MB | 3615 | 5.13 GB | +| 64 MB | 3596 | 5.13 GB | +| 128 MB | 3573 | 5.13 GB | +| 256 MB | 3539 | 5.13 GB | +| 512 MB | 3518 | 5.14 GB | + +Knee ≈ 256 MB — the curve is *monotonically* decreasing because +each "unit of work" allocates more (per-file bytevector + match +result lists + intel/clamav lookup tables), so amortising GC over +more files keeps paying off. RSS is flat across the sweep — the +peak is bounded by per-worker liveset, not trip-bytes. + +**Lesson**: pick the knee from a sweep on your own workload. Don't +copy a value from a synthetic — the synthetic was deliberately +lean (one allocation per iter) and its 64 MB sweet spot doesn't +generalise to scanners that allocate richer per-file structures. + ## Putting them together A worked example combining all four modules: a multi-threaded file