docs: add measured jerboa-db vs Datomic benchmark + scripts
ober
bb1c219236ff43e3bc49517c15c669c022a0f4ce
new file mode 100644 --- /dev/null +++ b/benchmarks/datomic-bench.clj @@ -0,0 +1,84 @@ +;; Datomic (mem peer) port of the jerboa-db mbrainz benchmark, 1% scale. +;; Equivalent dataset (same counts/distributions, clean artist refs) + Q1-Q8. +(require '[datomic.api :as d]) + +(def schema + [{:db/ident :artist/gid :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/index true} + {:db/ident :artist/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :artist/sortName :db/valueType :db.type/string :db/cardinality :db.cardinality/one} + {:db/ident :artist/type :db/valueType :db.type/string :db/cardinality :db.cardinality/one} + {:db/ident :artist/country :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :artist/startYear :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :release/gid :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/unique :db.unique/identity :db/index true} + {:db/ident :release/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :release/artists :db/valueType :db.type/ref :db/cardinality :db.cardinality/many} + {:db/ident :release/year :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :release/status :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :release/country :db/valueType :db.type/string :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :track/name :db/valueType :db.type/string :db/cardinality :db.cardinality/one} + {:db/ident :track/position :db/valueType :db.type/long :db/cardinality :db.cardinality/one} + {:db/ident :track/duration :db/valueType :db.type/long :db/cardinality :db.cardinality/one :db/index true} + {:db/ident :track/artists :db/valueType :db.type/ref :db/cardinality :db.cardinality/many}]) + +(def rng (java.util.Random. 42)) +(defn pick [c] (nth c (.nextInt rng (count c)))) +(defn rint [lo hi] (+ lo (.nextInt rng (- hi lo)))) +(def countries ["US" "GB" "DE" "FR" "JP" "CA" "AU" "SE" "NL" "NO"]) +(def statuses ["Official" "Promotion" "Bootleg" "Pseudo-Release"]) +(def types ["Group" "Person" "Orchestra" "Choir" "Other"]) +(def firsts ["James" "John" "Robert" "Michael" "William" "David" "Richard" "Charles" "Mary" "Patricia" "Jennifer" "Linda" "Paul" "George" "Ringo" "Keith" "Mick" "Bob" "Eric" "Jimmy" "Roger"]) +(def lasts ["Smith" "Johnson" "Williams" "Jones" "Brown" "Davis" "Miller" "Wilson" "Moore" "Taylor" "Young" "Page" "Plant" "Clapton" "Richards" "Jagger" "Dylan"]) +(def bw ["The" "Dark" "Electric" "Blue" "Black" "Red" "Iron" "Golden" "Silver" "Wild" "Stone" "Fire" "Night"]) +(def bn ["Kings" "Stars" "Lights" "Birds" "Wolves" "Dragons" "Phoenix" "Riders" "Angels" "Heroes" "Legends"]) +(def aadj ["Dark" "Blue" "Eternal" "Lost" "Rising" "Broken" "Silent" "Electric" "Wild" "Golden" "Sacred"]) +(def anoun ["Side" "Road" "Dream" "Fire" "Night" "Sky" "Stone" "River" "Soul" "Heart" "Mind" "World" "Light"]) +(def tv ["Running" "Flying" "Dancing" "Crying" "Falling" "Rising" "Burning" "Fading"]) +(def tn ["Away" "Home" "Free" "Wild" "Alone" "Together" "Higher" "Deeper"]) +(defn aname [] (if (< (.nextInt rng 3) 1) (str (pick bw) " " (pick bn)) (str (pick firsts) " " (pick lasts)))) +(defn uuid [] (str (java.util.UUID/randomUUID))) + +(defn artist-tx [i] + (let [a (str "a" i) nm (aname)] + (concat + [{:db/id a :artist/gid (uuid) :artist/name nm :artist/sortName nm + :artist/type (pick types) :artist/country (pick countries) :artist/startYear (rint 1900 2020)}] + (for [r (range 5)] + {:db/id (str "r" i "-" r) :release/gid (uuid) :release/name (str (pick aadj) " " (pick anoun)) + :release/artists a :release/year (rint 1960 2024) :release/status (pick statuses) :release/country (pick countries)}) + (for [r (range 5) t (range 10)] + {:track/name (str (pick tv) " " (pick tn) " " t) :track/position (inc t) + :track/duration (rint 120000 480000) :track/artists a})))) + +(defn ms-of [thunk] (let [s (System/nanoTime)] (thunk) (/ (- (System/nanoTime) s) 1e6))) +(defn bench [label thunk] + (let [ts (sort (repeatedly 3 #(ms-of thunk))) + r (thunk) rows (if (number? r) r (count r))] + (println (format " %-34s %8.1f ms %s rows" label (nth ts 1) rows)))) + +(def n 2620) +(def uri "datomic:mem://mb") +(d/create-database uri) +(def conn (d/connect uri)) +(def load-ms + (ms-of (fn [] + @(d/transact conn schema) + (doseq [chunk (partition-all 18 (range n))] + @(d/transact conn (mapcat artist-tx chunk)))))) +(def dbv (d/db conn)) +(println (format "Datomic mem — loaded ~%d entities in %.0f ms" (* n 56) load-ms)) +(println (format " (artists %d, releases %d, tracks %d)" + (ffirst (d/q '[:find (count ?e) :where [?e :artist/gid]] dbv)) + (ffirst (d/q '[:find (count ?e) :where [?e :release/gid]] dbv)) + (ffirst (d/q '[:find (count ?e) :where [?e :track/duration]] dbv)))) +(def sname (ffirst (d/q '[:find ?n :where [_ :artist/name ?n]] dbv))) +(def seid (ffirst (d/q '[:find ?e :where [?e :artist/name _]] dbv))) + +(bench "Q1 artist exact name lookup" #(d/q '[:find ?e :in $ ?n :where [?e :artist/name ?n]] dbv sname)) +(bench "Q2 releases by artist name" #(d/q '[:find ?r ?rn :in $ ?an :where [?a :artist/name ?an] [?r :release/artists ?a] [?r :release/name ?rn]] dbv sname)) +(bench "Q3 startYear < 1960" #(d/q '[:find ?e ?nm ?y :in $ ?th :where [?e :artist/startYear ?y] [?e :artist/name ?nm] [(< ?y ?th)]] dbv 1960)) +(bench "Q4 tracks>240s shared-artist" #(d/q '[:find ?tn ?d ?rn :in $ ?m :where [?t :track/duration ?d] [?t :track/name ?tn] [?t :track/artists ?a] [?r :release/artists ?a] [?r :release/name ?rn] [(> ?d ?m)]] dbv 240000)) +(bench "Q5 releases per country" #(d/q '[:find ?c (count ?r) :where [?r :release/country ?c]] dbv)) +(bench "Q6 releases for artist (rev)" #(d/q '[:find ?r ?rn :in $ ?a :where [?r :release/artists ?a] [?r :release/name ?rn]] dbv seid)) +(bench "Q7 pull artist attrs" #(d/pull dbv [:artist/name :artist/type :artist/country :artist/startYear] seid)) +(bench "Q8 avg track dur by status" #(d/q '[:find ?st (count ?t) (avg ?d) :where [?t :track/duration ?d] [?t :track/artists ?a] [?r :release/artists ?a] [?r :release/status ?st]] dbv)) +(println "DONE") new file mode 100644 --- /dev/null +++ b/benchmarks/logback-warn.xml @@ -0,0 +1,6 @@ +<configuration> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder><pattern>%msg%n</pattern></encoder> + </appender> + <root level="WARN"><appender-ref ref="STDOUT"/></root> +</configuration> new file mode 100644 --- /dev/null +++ b/docs/datomic-comparison.md @@ -0,0 +1,95 @@ +# jerboa-db vs Datomic — measured benchmark + +A same-machine, in-memory head-to-head on the MBrainz benchmark at **1% scale** +(2,620 artists / 13,100 releases / 131,000 tracks ≈ 147k entities). + +- **jerboa-db**: `connect ":memory:"` (the default B+-tree backend) via + `make mbrainz-quick`. Reports the Datalog path and the DuckDB columnar + fallback separately. +- **Datomic**: Pro 1.0.7622, `datomic:mem://` peer (in-process), via a Clojure + port of the same schema/queries (see *Reproduce* below). + +The datasets are **equivalent, not identical** — same entity counts, +attributes, and value distributions, generated from independent RNGs. The join +row-counts landed within ~1% (Q4: jerboa 428,096 vs Datomic 432,453), so query +selectivities are comparable. + +## Results (median ms; lower is better) + +| Query | jerboa Datalog | jerboa + DuckDB | Datomic | +|---|--:|--:|--:| +| **Load** (~147k entities) | **1136** | — | 1745 | +| Q1 — artist exact name lookup | 0 | — | 0.7 | +| Q2 — releases by artist name (2-hop) | 0 | — | 0.9 | +| Q3 — `startYear < 1960` (range) | **1** | — | 4.8 | +| Q4 — tracks > 240s on shared-artist releases (multi-join) | 1885 | 363 | **368** | +| Q5 — releases per country (group-by) | **1** | — | 9.3 | +| Q6 — reverse-ref releases for an artist | 0 | — | 0.4 | +| Q7 — pull artist attributes | 0 | — | 0.0 | +| Q8 — avg track duration by release status (join + agg) | 1314 | **12** | 566 | + +## Findings + +1. **OLTP / point / index queries (Q1–Q3, Q5–Q7): jerboa is at parity or + faster.** It loads ~1.5× faster than Datomic's `mem` peer and beats it on the + indexed scans (Q3 1 vs 5 ms, Q5 1 vs 9 ms); the rest are sub-millisecond on + both. The ported index machinery — B+-tree memory index, real-cardinality + planner, native group-by / pure-aggregate fast paths — holds up against + Datomic. + +2. **Raw Datalog multi-way joins (Q4, Q8): Datomic is faster** — ~5× on Q4 + (368 vs 1885 ms) and ~2.3× on Q8 (566 vs 1314 ms). Datomic's relational join + engine (with better join ordering) outclasses jerboa's nested-loop Datalog. + This is the real remaining engine gap. + +3. **jerboa's DuckDB fallback erases that gap on analytics** — Q4 ties + Datomic-Datalog (363 vs 368 ms) and Q8 beats it 47× (12 vs 566 ms). + +## Honest caveats + +- **Datomic's own OLAP path is Presto, which was *not* run.** Finding #3 compares + jerboa+DuckDB against Datomic-*Datalog*, not Datomic-*Presto*. Datalog is not + Datomic's intended path for these analytical shapes — it ships Presto for + exactly this, just as jerboa ships DuckDB. +- **Datomic `mem` peer** has no transactor / storage / peer-cache tier; a + production Datomic configuration behaves differently. `mem` is the closest + in-process analog to jerboa's `:memory:`. +- **Equivalent-not-identical data**: jerboa's loader scrambles `track/artists` + via batch timing; the Datomic port assigns `track/artists` = the track's + release artist (clean). Join row-counts matched anyway (~428k vs ~432k). +- Medians of 3 runs; JVM warmup not deeply controlled. Single machine, single + session. + +## Net + +jerboa-db is **genuinely competitive with Datomic on OLTP** at this scale, +**behind on Datalog multi-way joins** (where Datomic's engine wins 2–5×), and +**ahead on analytics only because it offloads to DuckDB** — the same strategy +Datomic uses with Presto. The gap worth closing is jerboa's Datalog join engine +(hash/merge joins, better ordering); the analytics gap is already handled by the +columnar fallback. + +## Reproduce + +**jerboa** (from the repo root): + +``` +make mbrainz-quick +``` + +**Datomic** (Pro 1.0.7622 distribution; JDK on PATH). The Clojure benchmark and +the quiet-logging config are preserved in this repo at +`benchmarks/datomic-bench.clj` and `benchmarks/logback-warn.xml`. Run from the +extracted distribution directory: + +``` +cd /path/to/datomic-pro-1.0.7622 +java -Dlogback.configurationFile=~/mine/jerboa-db/benchmarks/logback-warn.xml \ + -cp "peer-1.0.7622.jar:lib/*" clojure.main \ + ~/mine/jerboa-db/benchmarks/datomic-bench.clj +``` + +`benchmarks/datomic-bench.clj` creates a `datomic:mem://` database, transacts the +equivalent schema, generates the 1% synthetic dataset, and times Q1–Q8 (median of +3). The schema and the eight query shapes mirror jerboa's +`benchmarks/mbrainz-bench.ss`.