benchmarks+docs: add Datomic-mining findings note + validation bench scripts
ober
959103623c0f095b293314fb9ff7d3744ecb9847
new file mode 100644 --- /dev/null +++ b/benchmarks/bt-bench.ss @@ -0,0 +1,62 @@ +;;; Isolated memory-index benchmark: load / scan / seek / count at scale. +;;; Run: scheme --libdirs "lib:~/mine/jerboa/lib" --script /tmp/bt-bench.ss [N] +(import (jerboa prelude) + (jerboa-db datom) + (jerboa-db index protocol) + (jerboa-db index memory)) + +(def args (cdr (command-line))) +(def N (if (pair? args) (string->number (car args)) 500000)) +(def NE (max 1 (quotient N 4))) ;; entity space (≈4 datoms/entity) +(def M 20000) ;; number of probe queries +(def BIG 1073741823) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (elapsed start) (- (now-ms) start)) + +;; deterministic LCG +(def seed (list 123456789)) +(def (rnd m) + (let ([s (modulo (+ (* (car seed) 1103515245) 12345) 2147483648)]) + (set-car! seed s) + (modulo s m))) + +(def is (make-mem-index-set)) +(def eavt (index-set-eavt is)) + +(printf "N=~a datoms, NE=~a entities, M=~a probes~n" N NE M) + +;; 1. LOAD +(let ([t0 (now-ms)]) + (let loop ([i 0]) + (when (< i N) + (dbi-add! eavt (make-datom (rnd NE) 1 i 1 #t)) + (loop (+ i 1)))) + (printf " load: ~a ms~n" (inexact->exact (round (elapsed t0))))) + +;; 2. FULL SCAN (forces final flush + walks every datom in order) +(let ([t0 (now-ms)]) + (let ([n (length (dbi-datoms eavt))]) + (printf " full-scan: ~a ms (~a datoms)~n" + (inexact->exact (round (elapsed t0))) n))) + +;; 3. ENTITY SEEK (point lookup of all datoms for a random entity) +(let ([t0 (now-ms)]) + (let loop ([i 0] [acc 0]) + (if (< i M) + (loop (+ i 1) (+ acc (length (dbi-seek eavt (rnd NE) #f #f #f)))) + (printf " seek x~a: ~a ms (~a hits)~n" + M (inexact->exact (round (elapsed t0))) acc)))) + +;; 4. RANGE COUNT (order-statistic count of a random entity's datoms) +(let ([t0 (now-ms)]) + (let loop ([i 0] [acc 0]) + (if (< i M) + (let* ([e (rnd NE)] + [lo (make-datom e 0 +min-val+ 0 #t)] + [hi (make-datom e BIG +max-val+ BIG #t)]) + (loop (+ i 1) (+ acc (dbi-count eavt lo hi)))) + (printf " count x~a: ~a ms (~a total)~n" + M (inexact->exact (round (elapsed t0))) acc)))) new file mode 100644 --- /dev/null +++ b/benchmarks/card-demo.ss @@ -0,0 +1,35 @@ +;;; Planner per-(attr,value) selectivity demo. +;;; Two plain (non-unique) string attrs: item/cat has 2 values (~N/2 matches), +;;; item/tag is unique-per-entity (1 match). Query is written in the BAD order +;;; (common clause first). Old planner scores both value clauses 60 -> keeps the +;;; bad order; new planner counts values -> puts the selective clause first. +(import (jerboa prelude) + (jerboa-db core)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) + +(def N 60000) +(def M 300) + +(def conn (connect ":memory:")) +(transact! conn + (list '((db/ident . item/cat) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + '((db/ident . item/tag) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)))) +(transact! conn + (for/collect ([i (in-range N)]) + `((item/cat . ,(if (even? i) "A" "B")) + (item/tag . ,(str "t" i))))) + +(def d (db conn)) +;; bad order on purpose: common clause (cat="A") written first +(def query '((find ?e) (where (?e item/cat "A") (?e item/tag "t41234")))) + +(displayln "result rows: " (length (q query d))) +(displayln "plan: " (explain-query query d)) + +(let ([t0 (now-ms)]) + (dotimes (_ M) (q query d)) + (displayln "q x" M ": " (inexact->exact (round (- (now-ms) t0))) " ms")) +(close conn) new file mode 100644 --- /dev/null +++ b/benchmarks/cursor-bench.ss @@ -0,0 +1,33 @@ +;;; Early-exit demo: first-match / top-k via lazy cursor vs eager dbi-range. +(import (jerboa prelude) + (jerboa-db datom) + (jerboa-db index protocol) + (jerboa-db index memory)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) + +(def N 500000) +(def M 200) +(def is (make-mem-index-set)) +(def eavt (index-set-eavt is)) +(let loop ([i 0]) (when (< i N) (dbi-add! eavt (make-datom i 1 i 1 #t)) (loop (+ i 1)))) +(def lo (make-datom 0 0 +min-val+ 0 #t)) +(def hi (make-datom (greatest-fixnum) 100 +max-val+ (greatest-fixnum) #t)) +(dbi-count eavt lo hi) ;; flush so the buffer is empty for all runs + +(printf "N=~a, range covers all, M=~a reps~n" N M) + +(let ([t0 (now-ms)]) + (dotimes (_ M) (car (dbi-range eavt lo hi))) + (printf " dbi-range + car (materialize ~a): ~a ms~n" N (ms t0))) + +(let ([t0 (now-ms)]) + (dotimes (_ M) (car (stream-take (dbi-cursor eavt lo hi) 1))) + (printf " cursor first-match (early-exit): ~a ms~n" (ms t0))) + +(let ([t0 (now-ms)]) + (dotimes (_ M) (stream-take (dbi-cursor eavt lo hi) 10)) + (printf " cursor top-10 (early-exit): ~a ms~n" (ms t0))) new file mode 100644 --- /dev/null +++ b/benchmarks/group-bench.ss @@ -0,0 +1,46 @@ +;;; Native columnar group-by vs Datalog q (and a correctness cross-check). +(import (jerboa prelude) + (jerboa-db core) + (jerboa-db query group)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) +(def (sort-al al) (list-sort (lambda (x y) (string<? (car x) (car y))) al)) +(def (rows->al rows) (sort-al (map (lambda (r) (cons (car r) (cadr r))) rows))) + +(def N 200000) +(def G 50) + +(def conn (connect ":memory:")) +(transact! conn + (list '((db/ident . item/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + '((db/ident . item/dur) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)))) +(let ([t0 (now-ms)]) + (transact! conn + (for/collect ([i (in-range N)]) + `((item/country . ,(str "C" (modulo i G))) + (item/dur . ,(+ 60 (modulo (* i 7) 300)))))) + (printf "loaded ~a entities, ~a groups, in ~a ms~n" N G (ms t0))) + +(def d (db conn)) + +(printf "~n-- group-count (Q5-like: count per country) --~n") +(def nat-c (sort-al (group-count d 'item/country))) +(def dl-c (rows->al (q '((find ?c (count ?e)) (where (?e item/country ?c))) d))) +(printf " results match: ~a~n" (equal? nat-c dl-c)) +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (group-count d 'item/country) (loop (+ k 1)))) + (printf " native x10: ~a ms~n" (ms t0))) +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (q '((find ?c (count ?e)) (where (?e item/country ?c))) d) (loop (+ k 1)))) + (printf " datalog x10: ~a ms~n" (ms t0))) + +(printf "~n-- group sum (Q8-like: sum dur per country) --~n") +(def nat-s (sort-al (group-aggregate d 'item/country 'item/dur 'sum))) +(def dl-s (rows->al (q '((find ?c (sum ?dd)) (where (?e item/country ?c) (?e item/dur ?dd))) d))) +(printf " results match: ~a~n" (equal? nat-s dl-s)) +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (group-aggregate d 'item/country 'item/dur 'sum) (loop (+ k 1)))) + (printf " native x10: ~a ms~n" (ms t0))) +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (q '((find ?c (sum ?dd)) (where (?e item/country ?c) (?e item/dur ?dd))) d) (loop (+ k 1)))) + (printf " datalog x10: ~a ms~n" (ms t0))) +(close conn) new file mode 100644 --- /dev/null +++ b/benchmarks/limit-bench.ss @@ -0,0 +1,37 @@ +;;; :limit early-exit vs full materialization. +(import (jerboa prelude) (jerboa-db core)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) + +(def N 200000) +(def conn (connect ":memory:")) +(transact! conn + (list '((db/ident . item/kind) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + '((db/ident . item/n) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)))) +(transact! conn + (for/collect ([i (in-range N)]) + `((item/kind . ,(if (even? i) "a" "b")) (item/n . ,i)))) +(def d (db conn)) +(printf "N=~a~n" N) + +(let ([t0 (now-ms)]) + (let ([r (q '((find ?e) (where (?e item/n ?x))) d)]) + (printf " single, no limit (~a rows): ~a ms~n" (length r) (ms t0)))) +(let ([t0 (now-ms)]) + (let loop ([k 0] [last '()]) + (if (< k 100) + (loop (+ k 1) (q '((find ?e) (where (?e item/n ?x)) (limit 10)) d)) + (printf " single, limit 10 (~a rows) x100: ~a ms~n" (length last) (ms t0))))) + +(let ([t0 (now-ms)]) + (let ([r (q '((find ?e ?x) (where (?e item/kind "a") (?e item/n ?x))) d)]) + (printf " multi, no limit (~a rows): ~a ms~n" (length r) (ms t0)))) +(let ([t0 (now-ms)]) + (let loop ([k 0] [last '()]) + (if (< k 100) + (loop (+ k 1) (q '((find ?e ?x) (where (?e item/kind "a") (?e item/n ?x)) (limit 10)) d)) + (printf " multi, limit 10 (~a rows) x100: ~a ms~n" (length last) (ms t0))))) +(close conn) new file mode 100644 --- /dev/null +++ b/benchmarks/ref-bench.ss @@ -0,0 +1,41 @@ +;;; Cross-check + bench: ref-hop group-by (real Q8) native vs datalog q. +(import (jerboa prelude) + (jerboa-db core) + (jerboa-db query group)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) +(def (sort-al al) (list-sort (lambda (x y) (string<? (car x) (car y))) al)) +(def (rows->al rows) (sort-al (map (lambda (r) (cons (car r) (cadr r))) rows))) + +(def NR 2000) ;; releases +(def NT 100000) ;; tracks + +(def conn (connect ":memory:")) +(transact! conn + (list '((db/ident . release/status) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + '((db/ident . track/release) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/one)) + '((db/ident . track/dur) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)))) +(transact! conn + (for/collect ([i (in-range NR)]) `((release/status . ,(str "S" (modulo i 4)))))) +(def rids (map car (q '((find ?r) (where (?r release/status ?s))) (db conn)))) +(let ([t0 (now-ms)]) + (transact! conn + (for/collect ([i (in-range NT)]) + `((track/release . ,(list-ref rids (modulo i NR))) + (track/dur . ,(+ 60 (modulo (* i 7) 300)))))) + (printf "loaded ~a releases + ~a tracks in ~a ms~n" NR NT (ms t0))) + +(def d (db conn)) +(def nat (sort-al (group-aggregate-via-ref d 'track/release 'release/status 'track/dur 'sum))) +(def dl (rows->al (q '((find ?s (sum ?dd)) + (where (?t track/release ?r) (?r release/status ?s) (?t track/dur ?dd))) d))) +(printf "ref-hop Q8: results match: ~a groups=~a~n" (equal? nat dl) (length nat)) + +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (group-aggregate-via-ref d 'track/release 'release/status 'track/dur 'sum) (loop (+ k 1)))) + (printf " native x10: ~a ms~n" (ms t0))) +(let ([t0 (now-ms)]) (let loop ([k 0]) (when (< k 10) (q '((find ?s (sum ?dd)) (where (?t track/release ?r) (?r release/status ?s) (?t track/dur ?dd))) d) (loop (+ k 1)))) + (printf " datalog x10: ~a ms~n" (ms t0))) +(close conn) new file mode 100644 --- /dev/null +++ b/benchmarks/seg-bench.ss @@ -0,0 +1,51 @@ +;;; Columnar segment vs per-datom FASL: size + aggregate speed (Q8-like). +(import (jerboa prelude) + (jerboa-db datom) + (jerboa-db segment)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) + +(def N 500000) +(def M 200) + +;; Q8-like: one attribute, ascending entities, double values (e.g. durations). +(def ds + (let loop ([i (- N 1)] [acc '()]) + (if (< i 0) acc + (loop (- i 1) + (cons (make-datom i 9 (exact->inexact (+ 60 (modulo (* i 7) 300))) + (+ 1 (quotient i 1000)) #t) + acc))))) +(def dv (list->vector ds)) +(def seg (make-segment ds)) + +(printf "N=~a, value type=~a~n" N (segment-vtype seg)) + +;; ---- Size: per-datom FASL (current durable encoding) vs columnar segment ---- +(def fasl-bytes + (fold-left + (lambda (acc d) + (let-values ([(p g) (open-bytevector-output-port)]) + (fasl-write d p) + (+ acc (bytevector-length (g))))) + 0 ds)) +(def seg-bytes (bytevector-length (segment->bytevector seg))) +(printf " size per-datom FASL: ~a bytes (~a B/datom)~n" + fasl-bytes (inexact->exact (round (/ fasl-bytes N)))) +(printf " size columnar segment: ~a bytes (~a B/datom) -> ~ax smaller~n" + seg-bytes (/ (* 1.0 seg-bytes) N) + (inexact->exact (round (/ (* 1.0 fasl-bytes) seg-bytes)))) + +;; ---- Aggregate speed: row-oriented (boxed) vs columnar (unboxed flvector) ---- +(let ([t0 (now-ms)]) + (dotimes (_ M) + (let loop ([i 0] [acc 0.0]) + (if (= i N) acc (loop (+ i 1) (+ acc (datom-v (vector-ref dv i))))))) + (printf " sum row-oriented (boxed) x~a: ~a ms~n" M (ms t0))) + +(let ([t0 (now-ms)]) + (dotimes (_ M) (segment-sum-v seg)) + (printf " sum columnar (unboxed) x~a: ~a ms~n" M (ms t0))) new file mode 100644 --- /dev/null +++ b/benchmarks/segtree-bench.ss @@ -0,0 +1,41 @@ +;;; Durable segment tree: metadata-count vs full-scan, and structural sharing. +(import (jerboa prelude) + (jerboa-db datom) + (jerboa-db index segtree)) + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) (/ (time-nanosecond t) 1e6)))) +(def (ms t0) (inexact->exact (round (- (now-ms) t0)))) + +(def N 200000) +(def SEG 1024) +(def ss (make-mem-segstore)) +(def cmp compare-datoms-eavt) +(def ds (for/collect ([i (in-range N)]) (make-datom i 1 i 1 #t))) + +(def build-start (now-ms)) +(def t (segtree-build ss ds cmp SEG)) +(printf "build ~a datoms -> ~a leaf segments in ~a ms~n" + N (length (segtree-live-ids t)) (ms build-start)) + +;; count over a large interior range: metadata vs full scan +(def lo (make-datom 1000 0 +min-val+ 0 #t)) +(def hi (make-datom 199000 (greatest-fixnum) +max-val+ (greatest-fixnum) #t)) +(def scan-start (now-ms)) +(def scan-count (length (segtree-range->list t ss lo hi))) +(printf " count via full scan: ~a (~a ms)~n" scan-count (ms scan-start)) +(def meta-start (now-ms)) +(let loop ([k 0] [v 0]) + (if (< k 100) + (loop (+ k 1) (segtree-count t ss lo hi)) + (printf " count via metadata x100: ~a (~a ms)~n" v (ms meta-start)))) + +;; structural sharing: a small add rewrites only the affected leaves +(def size0 (segstore-size ss)) +(def t1 (segtree-add t ss (list (make-datom 5 1 5 9 #t) + (make-datom 100500 1 100500 9 #t)) cmp SEG)) +(printf " structural sharing: ~a leaves; adding 2 datoms created ~a new segment(s)~n" + (length (segtree-live-ids t1)) (- (segstore-size ss) size0)) +(printf " old snapshot still ~a datoms; new tree ~a datoms~n" + (segtree-total t) (segtree-total t1)) new file mode 100644 --- /dev/null +++ b/docs/datomic-findings.md @@ -0,0 +1,140 @@ +# Datomic Pro internals → lessons for jerboa-db + +Reverse-engineered from `datomic-pro-1.0.7622/peer-1.0.7622.jar` (Clojure AOT +bytecode). Decompiled with **CFR** to `work/cfr-core/` (2008 files covering the +storage + index + query core). jadx was tried first but folds Clojure's +`$`-munged function classes into Java inner classes and silently drops them — +**use CFR with a dotted-FQN `--jarfilter`**, e.g. +`java -jar cfr.jar peer.jar --jarfilter 'datomic\.(btset|index|db|iter)\..*' --outputdir out`. + +Datomic is Clojure, not Java; "decompile to Java" gives readable algorithms but +machinery noise (`RT.var`, nulling locals). Function/type names survive intact. + +--- + +## The architecture in one paragraph + +Each of the 4 covering indexes (EAVT/AEVT/AVET/VAET) is **two layers merged at +read time**: a small in-memory index of recent datoms (a `btset` B+-tree) and a +large immutable **durable index tree stored as segments in a KV store**. A read +(`seek-datoms`) returns a *lazy* k-way sorted-merge cursor over both layers, then +wraps it in a streaming `windowed` filter that applies as-of/time and +assert/retract dedup. A periodic background **indexing job** merge-sorts the +memory layer into the durable tree, writes new immutable segments, and resets the +memory layer. Immutable segments addressed by id ⇒ every `db` value is an O(1) +snapshot with full structural sharing (this is what makes `as-of`, history, and +speculative `with` cheap). + +--- + +## 1. Memory index: B+-tree (`btset`), not red-black ⟵ highest-value + +`datomic.btset` (`work/cfr-core/datomic/btset/`). A persistent **B+-tree**: + +- `BTSet{cmp, cnt, root}`; `BTSetLeaf{cnt, cmp, ks=Object[]}` (≤16 keys, sorted); + `BTSetBranch{cmp, nks=Object[]}` interleaved `[child0,sep0,child1,sep1,…]`, + separator = min-key of the right child (routing keys, B+-tree style). +- Insert = path-copy down one root→leaf path; leaf/branch **split** on overflow + (width 16, half=8), bubbling a `BTSetSplit{left,k,right}` up; root split grows + height. Set semantics: `compare==0` ⇒ insert is a no-op (returns `this`). +- **Append fast-path** (`BTSetLeaf.conjoin`): inserting at the *end* of a *full* + leaf returns `BTSetSplit(this, k, newLeaf[k])` — a cheap right sibling with **no + array copy**. Makes in-order/sorted bulk insert near-linear. +- Iteration via explicit path-link cursors (`BTSetIter`/`BTSetIterLink` = node + + index + parent-link) ⇒ O(1)-amortized `next`/`prev`, no recursion. + +**jerboa-db today** (`lib/jerboa-db/index/memory.ss`): Okasaki **red-black tree**, +node = `#(color left key val right)`, fanout 2. + +**Lesson.** Replace the RB-tree with a width-32 B+-tree: +- Depth at 1M datoms: RB ≈ 40 vs B+-tree ≈ 4. Each insert path-copies flat arrays + (cache-friendly) instead of many 5-slot vectors + rotations ⇒ less GC, the + "allocation pressure" status.md worried about. +- The append fast-path compounds with sorting each tx batch into index order + before insert ⇒ fast bulk load. +- Chez has `vector`/`fxvector` + `(meta-cond)` — a direct port of `BTSetLeaf`/ + `BTSetBranch` is straightforward. Reference port: Tonsky's + `persistent-sorted-set` (open-source twin of this exact design; Datahike uses + it). This is the single biggest structural win available. + +## 2. Index reads are LAZY cursors, not lists ⟵ likely the Q4/Q8 fix + +`datomic.iter`: `Iter` is a cursor (`get`/`next`/`prev`); `merge-iters` composes +binary `MergeIter`s into a k-way merge (`least-index` picks the smaller head); +`iter$filter`/`take-while`/`drop-while`/`map` are all streaming. `seek-datoms` +returns this lazy iterable — nothing is materialized until consumed. + +**jerboa-db today** (`lib/jerboa-db/index/protocol.ss`): `dbi-range`/`dbi-seek` +return **fully materialized lists**. + +**Lesson.** Make the index protocol return lazy ordered cursors (Chez: a +closure/coroutine or a struct with `peek`/`advance`). Then `query/engine.ss` can +do **streaming merge-joins** and honor `LIMIT`/early-exit without building whole +result lists. Q4 (2.14M-row wide join) and Q8 (group-by) — the two queries you +offload to DuckDB — are exactly the ones penalized by list materialization. This +may shrink the DuckDB gap or remove the need for it on some queries. + +## 3. Durable segments are COLUMNAR (transposed) ⟵ attacks the analytics gap + +Durable index = a shallow 3-level B-tree in the KV store: +`RootNode{keydata, dirids→dirs}` → `DirNode{keydata, segids→segs, offsets[], +counts[]}` → leaf **segments**. Each `keydata`/segment is a **`TransposedData`** +(`datomic.index`): columns, not rows — `eas` (packed entity+attr), `vs` (values), +t/op columns, with **type-specialized primitive accessors** `getLongV/getDoubleV/ +getIntV/getFloatV` (no boxing for homogeneous columns). + +**Lesson.** When jerboa persists (`value-store.ss`, `index/leveldb.ss`, +`encoding.ss`), store each segment **column-transposed** with primitive value +columns. Buys: (a) compression (RLE/delta over sorted e/a runs), (b) fast scans, +(c) cardinality without decoding (next point). This is the columnar property you +currently rent from DuckDB — owning it narrows that gap and feeds the planner. + +## 4. Range cardinality is O(1) from the directory ⟵ feeds the planner + +`DirNode` carries `offsets[]` + `counts[]` per segment, so Datomic counts any +key-range (`bounded-count`, `seg-count-cumsum`) **without loading segments**. + +**jerboa-db** has `dbi-count` (good) + `query/planner.ss` + `stats.ss`. + +**Lesson.** Give the planner *true* selectivity, not heuristics: either augment +the memory tree to an order-statistic tree (store subtree size per node — trivial +on a B+-tree, just sum child counts) or read counts straight from columnar +segment metadata. Real per-`a` and per-`(a, v-range)` counts ⇒ correct clause +ordering, which is most of query performance. + +## 5. Four indexes, one tuple shape; selective AVET/VAET + +`IndexSet{eavt, avet, aevt, raet, fulltext}` (`raet` = the refs/VAET index). +`db.seek-datoms` reorders user components into a single internal `[e a v t]` +tuple per index, each with its own comparator, and **errors if you AVET-seek a +non-indexed attribute** (`hasAVET`). So AVET holds only `:db/index`/`:db/unique` +attrs and VAET only refs — keeping the selective indexes small. + +**Lesson.** Confirm `index/protocol.ss` keeps AVET restricted to indexed/unique +attrs and VAET to refs only (don't index every attribute's value). Normalize all +four to one `[e a v t]` datom layout + per-index comparator (you likely do). + +## 6. History/as-of is a streaming FILTER, not a second store + +Retractions live **inline** in the same indexes (op/added column + t). The +"current" db is the raw index iter wrapped by `windowed` + `filter-retractions`, +which dedup assert/retract pairs and cut at a `t`. `as-of`/`since`/`history` are +just different filters over the same segments. + +**Lesson.** Ensure `history.ss` is a filter over the shared indexes (inline +retractions), not a separate history store — that's what makes `as-of` and +`history` free and consistent. + +--- + +## Reading map (in `work/cfr-core/datomic/`) +- B+-tree: `btset/BTSetLeaf.java`, `btset/BTSetBranch.java`, `btset/BTSet.java` +- Lazy merge: `iter$merge_iters.java`, `iter$least_index.java`, `iter/MergeIter.java` +- Read path: `db$seek_datoms.java`, `db$seek_datoms$fn__12075.java` (`windowed`) +- Durable tree: `index/RootNode.java`, `index/DirNode.java`, `index/TransposedData.java`, `index/ITransposeData.java` +- Indexing job (mem→durable): `index$merge_one_index.java`, `index$build_one_seg.java`, `index$merge_db_STAR_.java` +- Db value: `db/IndexSet.java`, `db/Db.java` + +Priority for jerboa-db: **§1 (B+-tree) and §2 (lazy cursors) first** — biggest +wins, contained to `index/memory.ss` + `index/protocol.ss`. Then §3/§4 (columnar +segments + planner stats) to address the analytical queries structurally.