feat: MBrainz benchmark harness — all 8 queries running
ober
2bbbf5561afe2ae5e2a3dc660e3375b521814b13
--- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ CHEZ_EXT_DIR ?= $(HOME)/src CHEZ_EXT_LIBDIRS = $(CHEZ_EXT_DIR)/chez-lmdb:$(CHEZ_EXT_DIR)/chez-duckdb FULL_LIBDIRS = $(LIBDIRS):$(CHEZ_EXT_LIBDIRS) -.PHONY: test build clean check bench bench-quick +.PHONY: test build clean check bench bench-quick mbrainz mbrainz-quick # Run the core test suite (in-memory, no FFI deps) test: @@ -40,6 +40,14 @@ bench: bench-quick: $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/load-test.ss --quick +# MBrainz benchmark (8 standard queries, synthetic data at full scale) +mbrainz: + $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/mbrainz-bench.ss + +# MBrainz quick smoke test (1% scale, ~3 runs per query) +mbrainz-quick: + $(SCHEME) --libdirs "$(LIBDIRS)" --script benchmarks/mbrainz-bench.ss --quick + # Clean compiled artifacts clean: find lib -name "*.so" -delete new file mode 100644 --- /dev/null +++ b/benchmarks/mbrainz-bench.ss @@ -0,0 +1,369 @@ +#!/usr/bin/env scheme --libdirs lib:~/mine/jerboa/lib --script +;;; MBrainz benchmark runner for Jerboa-DB +;;; +;;; Runs the 8 standard Datahike MBrainz benchmark queries with timing. +;;; +;;; Usage: +;;; make mbrainz -- full scale, all 8 queries +;;; make mbrainz-quick -- 1% scale, quick smoke test +;;; scheme ... --script benchmarks/mbrainz-bench.ss [--quick] [--scale 0.1] + +(import (jerboa prelude) + (jerboa-db core)) + +;; ---- Config ---- + +(def *scale* 1.0) +(def *runs* 5) ;; repetitions per query for timing +(def *warmup* 1) ;; warmup runs (not counted) + +(def (parse-args! args) + (let loop ([as args]) + (cond + [(null? as) (void)] + [(string=? (car as) "--quick") + (set! *scale* 0.01) + (set! *runs* 3) + (loop (cdr as))] + [(and (string=? (car as) "--scale") (pair? (cdr as))) + (set! *scale* (string->number (cadr as))) + (loop (cddr as))] + [(and (string=? (car as) "--runs") (pair? (cdr as))) + (set! *runs* (string->number (cadr as))) + (loop (cddr as))] + [else (loop (cdr as))]))) + +;; ---- Timing ---- + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) + (/ (time-nanosecond t) 1e6)))) + +(def (time-median thunk n warmup) + ;; Run thunk n+warmup times, discard warmup, return (median-ms result) + (let loop ([i 0] [times '()] [last-result #f]) + (if (>= i (+ n warmup)) + (let* ([sorted (sort times <)] + [mid (quotient (length sorted) 2)] + [med (list-ref sorted mid)]) + (list med last-result)) + (let ([t0 (now-ms)]) + (let ([r (thunk)]) + (let ([ms (- (now-ms) t0)]) + (loop (+ i 1) + (if (>= i warmup) (cons ms times) times) + r))))))) + +;; ---- Formatting ---- + +(def (pad-right s n) + (let ([l (string-length s)]) + (if (>= l n) s (string-append s (make-string (- n l) #\space))))) + +(def (pad-left s n) + (let ([l (string-length s)]) + (if (>= l n) s (string-append (make-string (- n l) #\space) s)))) + +(def (fmt-ms ms) + (let* ([rounded (inexact->exact (round ms))] + [s (string-append (number->string rounded) " ms")]) + (pad-left s 9))) + +;; ---- Schema (inline) ---- + +(def (mbrainz-schema-ops) + (list + `((db/ident . abstract/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . artist/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . artist/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t) (db/fulltext . #t)) + `((db/ident . artist/sortName) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/type) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/gender) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . artist/startYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . artist/endYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . release/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t) (db/fulltext . #t)) + `((db/ident . release/artists) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . release/year) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/status) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . medium/release) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/format) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/position) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/tracks) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . track/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/fulltext . #t)) + `((db/ident . track/position) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . track/duration) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . track/artists) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . label/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . label/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . label/type) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . label/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . label/startYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . release/labels) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)))) + +;; ---- Synthetic data generation ---- + +(def *rng-state* 42424242424242) +(def (next-rand!) (set! *rng-state* (modulo (+ (* *rng-state* 6364136223846793005) 1442695040888963407) (expt 2 64))) *rng-state*) +(def (rand-int! lo hi) (+ lo (modulo (next-rand!) (- hi lo)))) +(def (rand-pick! lst) (list-ref lst (modulo (next-rand!) (length lst)))) +(def (make-uuid!) (let ([h (lambda () (let ([s (number->string (modulo (next-rand!) 65536) 16)]) (string-append (make-string (- 4 (string-length s)) #\0) s)))]) (string-append (h)(h) "-" (h) "-" (h) "-" (h) "-" (h)(h)(h)))) + +(def *countries* '("US" "GB" "DE" "FR" "JP" "CA" "AU" "SE" "NL" "NO")) +(def *statuses* '("Official" "Promotion" "Bootleg")) +(def *first-names* '("James" "John" "Robert" "Michael" "David" "Paul" "George" "Bob" "Eric" "Jimmy")) +(def *last-names* '("Smith" "Johnson" "Williams" "Jones" "Brown" "Davis" "Young" "Page" "Plant")) +(def *band-words* '("The" "Dark" "Electric" "Blue" "Black" "Iron" "Golden" "Wild" "Stone" "Fire")) +(def *band-nouns* '("Kings" "Stars" "Birds" "Wolves" "Angels" "Heroes" "Warriors" "Giants")) +(def *album-adjs* '("Dark" "Blue" "Eternal" "Lost" "Rising" "Broken" "Silent" "Electric" "Wild")) +(def *album-nouns* '("Side" "Road" "Dream" "Fire" "Night" "Sky" "Stone" "River" "Soul" "Heart")) +(def *artist-types* '("Group" "Person" "Orchestra")) + +(def (make-artist-name!) (if (< (modulo (next-rand!) 3) 1) (string-append (rand-pick! *band-words*) " " (rand-pick! *band-nouns*)) (string-append (rand-pick! *first-names*) " " (rand-pick! *last-names*)))) +(def (make-album-name!) (string-append (rand-pick! *album-adjs*) " " (rand-pick! *album-nouns*))) + +;; Flush batch via transact!, return resolved entity IDs +(def (flush-batch! conn batch) + (if (null? batch) + '() + (map cdr (tx-report-tempids (transact! conn (reverse batch)))))) + +(def (load-synthetic-data! conn scale batch-size) + (let ([num-artists (inexact->exact (round (* 262000 scale)))] + [releases-per-artist 5] + [tracks-per-artist 50]) ;; 5 releases × 10 tracks, stored flat per artist + + ;; ---- Schema ---- + (transact! conn (mbrainz-schema-ops)) + + ;; ---- Phase 1: artists — collect eids for later reference ---- + (let ([artist-eids '()]) + (let loop ([i 0] [batch '()]) + (cond + [(= i num-artists) + (set! artist-eids (append (flush-batch! conn batch) artist-eids))] + [else + (let* ([op `((db/id . ,(tempid)) + (artist/gid . ,(make-uuid!)) + (artist/name . ,(make-artist-name!)) + (artist/type . ,(rand-pick! *artist-types*)) + (artist/country . ,(rand-pick! *countries*)) + (artist/startYear . ,(rand-int! 1920 2010)))] + [b (cons op batch)]) + (if (>= (length b) batch-size) + (begin + (set! artist-eids (append (flush-batch! conn b) artist-eids)) + (loop (+ i 1) '())) + (loop (+ i 1) b)))])) + + ;; ---- Phase 2: releases — flat pass, batch by batch ---- + (let ([total-releases 0]) + (let loop ([eids artist-eids] [batch '()]) + (cond + [(null? eids) + (set! total-releases (+ total-releases (length batch))) + (flush-batch! conn batch)] + [else + (let* ([aeid (car eids)] + [new-ops (map (lambda (_) + `((db/id . ,(tempid)) + (release/gid . ,(make-uuid!)) + (release/name . ,(make-album-name!)) + (release/artists . ,aeid) + (release/year . ,(rand-int! 1950 2024)) + (release/status . ,(rand-pick! *statuses*)) + (release/country . ,(rand-pick! *countries*)))) + (iota releases-per-artist))] + [b (append new-ops batch)]) + (if (>= (length b) batch-size) + (begin + (set! total-releases (+ total-releases (length b))) + (flush-batch! conn b) + (loop (cdr eids) '())) + (loop (cdr eids) b)))])) + + ;; ---- Phase 3: tracks — flat pass, batch by batch ---- + (let ([total-tracks 0]) + (let loop ([eids artist-eids] [batch '()]) + (cond + [(null? eids) + (set! total-tracks (+ total-tracks (length batch))) + (flush-batch! conn batch)] + [else + (let* ([aeid (car eids)] + [new-ops (map (lambda (ti) + `((track/name . ,(string-append "Track " (number->string ti))) + (track/position . ,ti) + (track/duration . ,(rand-int! 120000 480000)) + (track/artists . ,aeid))) + (iota tracks-per-artist 1))] + [b (append new-ops batch)]) + (if (>= (length b) batch-size) + (begin + (set! total-tracks (+ total-tracks (length b))) + (flush-batch! conn b) + (loop (cdr eids) '())) + (loop (cdr eids) b)))])) + + + (list 'artists (length artist-eids) + 'releases total-releases + 'tracks total-tracks)))))) + +;; ---- The 8 MBrainz benchmark queries ---- + +;; Q1: Simple attribute lookup — find an artist by exact name +;; Tests: AVET point lookup +(def (q1 db name) + (q '((find ?e) + (in $ ?name) + (where (?e artist/name ?name))) + db name)) + +;; Q2: Two-clause join — find releases by artist name +;; Tests: AEVT scan + join +(def (q2 db artist-name) + (q '((find ?r ?rname) + (in $ ?aname) + (where (?a artist/name ?aname) + (?r release/artists ?a) + (?r release/name ?rname))) + db artist-name)) + +;; Q3: Range predicate — artists with startYear before threshold +;; Tests: AVET range scan with predicate pushdown +(def (q3 db year-threshold) + (q '((find ?e ?name ?year) + (in $ ?thresh) + (where (?e artist/startYear ?year) + (?e artist/name ?name) + ((< ?year ?thresh)))) + db year-threshold)) + +;; Q4: Multi-join — tracks with duration above threshold on releases by artist +;; Tests: deep join, 4 data patterns + predicate +(def (q4 db min-duration) + (q '((find ?track-name ?duration ?release-name) + (in $ ?min-dur) + (where (?t track/duration ?duration) + (?t track/name ?track-name) + (?t track/artists ?a) + (?r release/artists ?a) + (?r release/name ?release-name) + ((> ?duration ?min-dur)))) + db min-duration)) + +;; Q5: Aggregation — count releases per country +;; Tests: aggregation + grouping +(def (q5 db) + (q '((find ?country (count ?r)) + (where (?r release/country ?country))) + db)) + +;; Q6: Reverse ref navigation — find all releases for an artist entity +;; Tests: cardinality/many reverse lookup via VAET +(def (q6 db artist-eid) + (q '((find ?r ?rname) + (in $ ?artist) + (where (?r release/artists ?artist) + (?r release/name ?rname))) + db artist-eid)) + +;; Q7: Pull pattern — artist with all attributes +;; Tests: pull API +(def (q7 db artist-eid) + (pull db '[artist/name artist/type artist/country artist/startYear] artist-eid)) + +;; Q8: Complex aggregation — avg track duration by release status +;; Tests: aggregation + join + grouping +(def (q8 db) + (q '((find ?status (count ?t) (avg ?dur)) + (where (?t track/duration ?dur) + (?t track/artists ?a) + (?r release/artists ?a) + (?r release/status ?status))) + db)) + +;; ---- Benchmark runner ---- + +(def (run-benchmark conn sample-artist-eid sample-artist-name) + (let ([db (db conn)]) + + (displayln "") + (displayln "=== Jerboa-DB MBrainz Benchmark ===") + (displayln "") + (displayln (pad-right "Query" 55) (pad-left "Median" 10) (pad-left "Result" 15)) + (displayln (make-string 80 #\-)) + + (let ([bench (lambda (name label thunk) + (let* ([result (time-median thunk *runs* *warmup*)] + [ms (car result)] + [r (cadr result)] + [n (cond [(list? r) (length r)] + [(number? r) r] + [else 1])]) + (displayln (pad-right (string-append name ": " label) 55) + (fmt-ms ms) + (pad-left (string-append (number->string n) " rows") 15)) + (cons name ms)))]) + + (let ([results + (list + (bench "Q1" "Artist exact name lookup" + (lambda () (q1 db sample-artist-name))) + (bench "Q2" "Releases by artist name" + (lambda () (q2 db sample-artist-name))) + (bench "Q3" "Artists with startYear < 1960" + (lambda () (q3 db 1960))) + (bench "Q4" "Tracks > 240s on shared-artist releases" + (lambda () (q4 db 240000))) + (bench "Q5" "Count releases per country" + (lambda () (q5 db))) + (bench "Q6" "All releases for one artist (reverse ref)" + (lambda () (q6 db sample-artist-eid))) + (bench "Q7" "Pull artist attributes" + (lambda () (q7 db sample-artist-eid))) + (bench "Q8" "Avg track duration by release status" + (lambda () (q8 db))))]) + + (displayln (make-string 80 #\-)) + (let ([total (apply + (map cdr results))]) + (displayln (pad-right "Total" 55) (fmt-ms total))) + (displayln "") + results)))) + +;; ---- Main ---- + +(parse-args! (cdr (command-line))) + +(displayln "MBrainz Benchmark — scale=" *scale* " runs=" *runs*) +(displayln "") + +(def conn (connect ":memory:")) + +;; Load data +(display "Loading data... ") +(def t0 (now-ms)) +(def load-stats (load-synthetic-data! conn *scale* 500)) +(displayln "done in " (inexact->exact (round (- (now-ms) t0))) " ms") +(displayln " " load-stats) + +;; Find a sample artist for queries that need one +(def sample-artist + (let ([rows (q '((find ?e ?name) + (where (?e artist/name ?name))) + (db conn))]) + (if (pair? rows) + (car rows) + (list 1 "Unknown Artist")))) + +(def sample-artist-eid (car sample-artist)) +(def sample-artist-name (cadr sample-artist)) + +(displayln " Sample artist: eid=" sample-artist-eid " name=" sample-artist-name) + +;; Run benchmark +(run-benchmark conn sample-artist-eid sample-artist-name) new file mode 100644 --- /dev/null +++ b/benchmarks/mbrainz-loader.ss @@ -0,0 +1,327 @@ +#!/usr/bin/env scheme --libdirs lib:~/mine/jerboa/lib --script +;;; MBrainz bulk data loader for Jerboa-DB +;;; +;;; Loads either actual MBrainz EDN files or generates synthetic data at scale. +;;; +;;; Usage: +;;; make bench -- full scale (6.6M entities) +;;; scheme ... --script benchmarks/mbrainz-loader.ss --quick -- 66K entities +;;; scheme ... --script benchmarks/mbrainz-loader.ss --scale 0.1 +;;; +;;; Returns a connection ready for benchmarking. + +(import (jerboa prelude) + (jerboa-db core)) + +;; ---- Import schema installer ---- +;; (loaded inline since we can't import .ss files directly) + +;; ---- Config ---- + +(def *scale* 1.0) ;; 1.0 = full MBrainz scale (~6.6M entities) +(def *batch-size* 1000) + +(def (parse-args! args) + (let loop ([as args]) + (cond + [(null? as) (void)] + [(string=? (car as) "--quick") + (set! *scale* 0.01) + (loop (cdr as))] + [(and (string=? (car as) "--scale") (pair? (cdr as))) + (set! *scale* (string->number (cadr as))) + (loop (cddr as))] + [else (loop (cdr as))]))) + +;; ---- Timing ---- + +(def (now-ms) + (let ([t (current-time 'time-monotonic)]) + (+ (* 1000.0 (time-second t)) + (/ (time-nanosecond t) 1e6)))) + +;; ---- Pseudo-random data generation ---- +;; Simple LCG for reproducible synthetic data + +(def *rng-state* 12345678901234567) + +(def (next-rand!) + (set! *rng-state* (modulo (+ (* *rng-state* 6364136223846793005) 1442695040888963407) + (expt 2 64))) + *rng-state*) + +(def (rand-int! lo hi) + (+ lo (modulo (next-rand!) (- hi lo)))) + +(def (rand-pick! lst) + (list-ref lst (modulo (next-rand!) (length lst)))) + +(def (make-uuid!) + ;; Generate a fake UUID string + (def (hex4) (let ([n (modulo (next-rand!) 65536)]) + (let ([s (number->string n 16)]) + (string-append (make-string (- 4 (string-length s)) #\0) s)))) + (string-append (hex4) (hex4) "-" (hex4) "-" (hex4) "-" (hex4) "-" (hex4) (hex4) (hex4))) + +;; ---- Synthetic data ---- + +(def *artist-types* '("Group" "Person" "Orchestra" "Choir" "Other")) +(def *genders* '("Male" "Female" "Other")) +(def *countries* '("US" "GB" "DE" "FR" "JP" "CA" "AU" "SE" "NL" "NO")) +(def *statuses* '("Official" "Promotion" "Bootleg" "Pseudo-Release")) +(def *formats* '("CD" "Vinyl" "Digital Media" "Cassette" "DVD" "Blu-ray")) +(def *label-types* '("Original Production" "Bootleg Production" "Reissue Production" "Publisher")) + +(def *first-names* '("James" "John" "Robert" "Michael" "William" "David" "Richard" "Charles" + "Mary" "Patricia" "Jennifer" "Linda" "Barbara" "Susan" "Jessica" "Sarah" + "Paul" "George" "Ringo" "Keith" "Mick" "Bob" "Eric" "Jimmy" "Roger")) +(def *last-names* '("Smith" "Johnson" "Williams" "Jones" "Brown" "Davis" "Miller" "Wilson" + "Moore" "Taylor" "Anderson" "Jackson" "White" "Harris" "Martin" "Garcia" + "Young" "Page" "Plant" "Clapton" "Richards" "Jagger" "Dylan")) +(def *band-words* '("The" "Dark" "Electric" "Blue" "Black" "Red" "Iron" "Golden" "Silver" + "Wild" "Stone" "Fire" "Night" "Dream" "Magic" "Cosmic" "Atomic")) +(def *band-nouns* '("Kings" "Stars" "Lights" "Birds" "Wolves" "Dragons" "Phoenix" "Riders" + "Angels" "Demons" "Heroes" "Legends" "Warriors" "Giants" "Travelers")) + +(def *album-adjectives* '("Dark" "Blue" "Eternal" "Lost" "Rising" "Broken" "Silent" "Electric" + "Wild" "Golden" "Sacred" "Hollow" "Ancient" "Burning" "Fading")) +(def *album-nouns* '("Side" "Road" "Dream" "Fire" "Night" "Sky" "Stone" "River" "Soul" + "Heart" "Mind" "World" "Time" "Space" "Light" "Shadow" "Voice")) + +(def *track-verbs* '("Running" "Flying" "Dancing" "Crying" "Falling" "Rising" "Burning" "Fading")) +(def *track-nouns* '("Away" "Home" "Free" "Wild" "Alone" "Together" "Higher" "Deeper")) + +(def (make-artist-name!) + (if (< (modulo (next-rand!) 3) 1) + ;; Band name + (string-append (rand-pick! *band-words*) " " (rand-pick! *band-nouns*)) + ;; Person name + (string-append (rand-pick! *first-names*) " " (rand-pick! *last-names*)))) + +(def (make-album-name!) + (string-append (rand-pick! *album-adjectives*) " " (rand-pick! *album-nouns*))) + +(def (make-track-name! n) + (string-append (rand-pick! *track-verbs*) " " (rand-pick! *track-nouns*) + " " (number->string n))) + +;; ---- Scale targets ---- +;; MBrainz full scale: +;; ~262K artists, ~1.4M releases, ~1.4M media, ~4.8M tracks = ~7.9M total entities +;; (The 6.6M figure is the commonly cited entity count for a subset) + +(def (entity-counts scale) + ;; Returns (artists releases-per-artist tracks-per-release) + (let ([artists (inexact->exact (round (* 262000 scale)))]) + (list artists 5 10))) ;; avg 5 releases/artist, 10 tracks/release + +;; ---- Batch loading ---- + +(def (batch-transact! conn ops progress-label total-batches batch-num) + (transact! conn ops) + ;; Show progress every 100 batches + (when (= 0 (modulo batch-num 100)) + (display (string-append "\r " progress-label ": " + (number->string (* batch-num *batch-size*)) + "/" (number->string (* total-batches *batch-size*)) + " entities ")) + (flush-output-port (current-output-port)))) + +;; ---- Schema installation (inline) ---- + +(def (mbrainz-schema-ops) + (list + `((db/ident . abstract/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . artist/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . artist/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t) (db/fulltext . #t)) + `((db/ident . artist/sortName) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/type) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/gender) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . artist/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . artist/startYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . artist/endYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . release/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t) (db/fulltext . #t)) + `((db/ident . release/artists) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . release/year) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/status) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . release/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . medium/release) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/format) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/position) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . medium/tracks) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . track/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/fulltext . #t)) + `((db/ident . track/position) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . track/duration) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . track/artists) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)) + `((db/ident . label/gid) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/unique . db.unique/identity) (db/index . #t)) + `((db/ident . label/name) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . label/type) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one)) + `((db/ident . label/country) (db/valueType . db.type/string) (db/cardinality . db.cardinality/one) (db/index . #t)) + `((db/ident . label/startYear) (db/valueType . db.type/long) (db/cardinality . db.cardinality/one)) + `((db/ident . release/labels) (db/valueType . db.type/ref) (db/cardinality . db.cardinality/many)))) + +;; ---- Main loader ---- + +(def (load-mbrainz! conn . opts) + (let ([scale (if (null? opts) *scale* (car opts))]) + (let ([t0 (now-ms)]) + + ;; 1. Install schema + (display "Installing MBrainz schema... ") + (transact! conn (mbrainz-schema-ops)) + (displayln "done (" (inexact->exact (round (- (now-ms) t0))) " ms)") + + ;; 2. Load artists + (let-values ([(num-artists releases-per-artist tracks-per-release) + (apply values (entity-counts scale))]) + (displayln "Loading " num-artists " artists (" + (inexact->exact (round (* num-artists releases-per-artist))) " releases, " + (inexact->exact (round (* num-artists releases-per-artist tracks-per-release))) " tracks)...") + + ;; Phase A: Load artists in batches, collect eids + (let ([t1 (now-ms)] + [artist-eids (make-vector num-artists 0)]) + + (let loop ([i 0] [batch '()] [batch-count 0]) + (cond + [(= i num-artists) + ;; Flush final batch + (when (pair? batch) + (let* ([tid-list (map (lambda (op) (cdr (assq 'db/id op))) batch)] + [report (transact! conn (reverse batch))] + [tempids (tx-report-tempids report)]) + (for-each + (lambda (tid-pair) + (let* ([tid (car tid-pair)] + [eid (cdr tid-pair)] + [idx (- (length tid-list) 1 (length (memv tid tid-list)))]) + (when (< idx num-artists) + (vector-set! artist-eids idx eid)))) + tempids)))] + [else + (let* ([tid (tempid)] + [name (make-artist-name!)] + [op `((db/id . ,tid) + (artist/gid . ,(make-uuid!)) + (artist/name . ,name) + (artist/sortName . ,name) + (artist/type . ,(rand-pick! *artist-types*)) + (artist/country . ,(rand-pick! *countries*)) + (artist/startYear . ,(rand-int! 1900 2020)))]) + [new-batch (cons op batch)]) + (if (= (length new-batch) *batch-size*) + (let* ([tid-list (map (lambda (o) (cdr (assq 'db/id o))) new-batch)] + [report (transact! conn (reverse new-batch))] + [tempids (tx-report-tempids report)]) + ;; Record artist eids + (let ([batch-start (* batch-count *batch-size*)]) + (for-each + (lambda (tid-pair) + (let* ([tid (car tid-pair)] + [eid (cdr tid-pair)]) + (let find-idx ([tl tid-list] [j 0]) + (cond + [(null? tl) (void)] + [(equal? tid (car tl)) + (let ([idx (+ batch-start j)]) + (when (< idx num-artists) + (vector-set! artist-eids idx eid)))] + [else (find-idx (cdr tl) (+ j 1))])))) + tempids)) + (when (= 0 (modulo (+ batch-count 1) 50)) + (display (string-append "\r Artists: " + (number->string (* (+ batch-count 1) *batch-size*)) + "/" (number->string num-artists) " ")) + (flush-output-port (current-output-port))) + (loop (+ i 1) '() (+ batch-count 1))) + (loop (+ i 1) new-batch batch-count)))])) + + (displayln "\r Artists loaded in " (inexact->exact (round (- (now-ms) t1))) " ms ") + + ;; Phase B: Load releases + tracks + (let ([t2 (now-ms)] + [release-batch '()] + [track-batch '()] + [total-releases 0] + [total-tracks 0]) + + ;; For each artist, create releases + (let artist-loop ([ai 0]) + (when (< ai num-artists) + (let ([artist-eid (vector-ref artist-eids ai)]) + (when (> artist-eid 0) + (let release-loop ([ri 0]) + (when (< ri releases-per-artist) + (let* ([rel-tid (tempid)] + [rel-year (rand-int! 1960 2024)] + [rel-op `((db/id . ,rel-tid) + (release/gid . ,(make-uuid!)) + (release/name . ,(make-album-name!)) + (release/artists . ,artist-eid) + (release/year . ,rel-year) + (release/status . ,(rand-pick! *statuses*)) + (release/country . ,(rand-pick! *countries*)))]) + ;; Add to release batch + (set! release-batch (cons rel-op release-batch)) + (set! total-releases (+ total-releases 1)) + + ;; Flush release batch when full + (when (>= (length release-batch) *batch-size*) + (let* ([report (transact! conn (reverse release-batch))] + [tempids (tx-report-tempids report)]) + ;; Build rel-eid map for track creation + (let ([rel-eids (map cdr tempids)]) + ;; Create tracks for each release in this batch + (for-each + (lambda (rel-eid) + (let track-loop ([ti 0]) + (when (< ti tracks-per-release) + (let ([trk-op `((track/name . ,(make-track-name! ti)) + (track/position . ,(+ ti 1)) + (track/duration . ,(rand-int! 120000 480000)) + (track/artists . ,artist-eid))]) + (set! track-batch (cons trk-op track-batch)) + (set! total-tracks (+ total-tracks 1)) + (when (>= (length track-batch) *batch-size*) + (transact! conn (reverse track-batch)) + (set! track-batch '()))) + (track-loop (+ ti 1))))) + rel-eids))) + (set! release-batch '())) + (when (= 0 (modulo total-releases (* *batch-size* 10))) + (display (string-append "\r Releases: " + (number->string total-releases) " Tracks: " + (number->string total-tracks) " ")) + (flush-output-port (current-output-port))))) + (release-loop (+ ri 1)))))) + (artist-loop (+ ai 1)))) + + ;; Flush remaining batches + (when (pair? release-batch) + (transact! conn (reverse release-batch))) + (when (pair? track-batch) + (transact! conn (reverse track-batch))) + + (displayln "\r Releases: " total-releases " Tracks: " total-tracks + " (" (inexact->exact (round (- (now-ms) t2))) " ms) ") + + (let ([total-ms (inexact->exact (round (- (now-ms) t0)))]) + (displayln "Load complete: " (+ num-artists total-releases total-tracks) + " entities in " total-ms " ms" + " (" (inexact->exact (round (/ (* (+ num-artists total-releases total-tracks) 1000.0) total-ms))) + " entities/sec)") + (list 'artists num-artists + 'releases total-releases + 'tracks total-tracks + 'total-ms total-ms)))))))) + +;; ---- Main (if run as script) ---- + +(parse-args! (cdr (command-line))) + +(displayln "MBrainz Loader — scale=" *scale* " batch-size=" *batch-size*) +(def conn (connect ":memory:")) +(load-mbrainz! conn *scale*) +(let ([stats (db-stats conn)]) + (displayln "DB stats: " stats)) new file mode 100644 --- /dev/null +++ b/benchmarks/mbrainz-schema.ss @@ -0,0 +1,263 @@ +#!/usr/bin/env scheme --libdirs lib:~/mine/jerboa/lib --script +;;; MBrainz schema definition for Jerboa-DB +;;; +;;; All attribute definitions for the MusicBrainz dataset. +;;; Use: (import (jerboa-db core)) then call (install-mbrainz-schema! conn) + +(import (jerboa prelude) + (jerboa-db core)) + +;; ---- Schema definition ---- +;; Returns a list of tx-op alists to be passed to (transact! conn ops) + +(def (mbrainz-schema-ops) + (list + ;; ---- Abstract entity (artist / label base) ---- + `((db/ident . abstract/gid) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/unique . db.unique/identity) + (db/index . #t) + (db/doc . "MusicBrainz globally unique identifier (UUID string)")) + + `((db/ident . abstract/name) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Primary name of the entity")) + + ;; ---- Artist ---- + `((db/ident . artist/gid) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/unique . db.unique/identity) + (db/index . #t) + (db/doc . "Artist MusicBrainz GUID")) + + `((db/ident . artist/name) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/fulltext . #t) + (db/doc . "Artist name")) + + `((db/ident . artist/sortName) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Artist sort name (e.g. 'Beatles, The')")) + + `((db/ident . artist/type) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Artist type: person, group, orchestra, choir, character, other")) + + `((db/ident . artist/gender) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Artist gender: male, female, other")) + + `((db/ident . artist/country) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "ISO 3166-1 country code")) + + `((db/ident . artist/startYear) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Year the artist started activity")) + + `((db/ident . artist/startMonth) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Month the artist started activity")) + + `((db/ident . artist/startDay) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Day the artist started activity")) + + `((db/ident . artist/endYear) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Year the artist ended activity")) + + `((db/ident . artist/endMonth) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Month the artist ended activity")) + + `((db/ident . artist/endDay) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Day the artist ended activity")) + + ;; ---- Release ---- + `((db/ident . release/gid) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/unique . db.unique/identity) + (db/index . #t) + (db/doc . "Release MusicBrainz GUID")) + + `((db/ident . release/name) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/fulltext . #t) + (db/doc . "Release name")) + + `((db/ident . release/artists) + (db/valueType . db.type/ref) + (db/cardinality . db.cardinality/many) + (db/doc . "Artists credited on this release")) + + `((db/ident . release/year) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Release year")) + + `((db/ident . release/month) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Release month")) + + `((db/ident . release/day) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Release day")) + + `((db/ident . release/status) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Release status: official, promotion, bootleg, pseudo-release")) + + `((db/ident . release/country) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Release country ISO code")) + + `((db/ident . release/barcode) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "EAN/UPC barcode")) + + ;; ---- Medium ---- + `((db/ident . medium/release) + (db/valueType . db.type/ref) + (db/cardinality . db.cardinality/one) + (db/doc . "The release this medium belongs to")) + + `((db/ident . medium/format) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Medium format: CD, vinyl, digital media, etc.")) + + `((db/ident . medium/position) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Disc number within the release")) + + `((db/ident . medium/trackCount) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Number of tracks on this medium")) + + `((db/ident . medium/tracks) + (db/valueType . db.type/ref) + (db/cardinality . db.cardinality/many) + (db/doc . "Tracks on this medium")) + + ;; ---- Track ---- + `((db/ident . track/name) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/fulltext . #t) + (db/doc . "Track name")) + + `((db/ident . track/position) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Track position on the medium")) + + `((db/ident . track/duration) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Track duration in milliseconds")) + + `((db/ident . track/artists) + (db/valueType . db.type/ref) + (db/cardinality . db.cardinality/many) + (db/doc . "Artists performing on this track")) + + ;; ---- Label ---- + `((db/ident . label/gid) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/unique . db.unique/identity) + (db/index . #t) + (db/doc . "Label MusicBrainz GUID")) + + `((db/ident . label/name) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Label name")) + + `((db/ident . label/sortName) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Label sort name")) + + `((db/ident . label/type) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/doc . "Label type: original production, bootleg production, etc.")) + + `((db/ident . label/country) + (db/valueType . db.type/string) + (db/cardinality . db.cardinality/one) + (db/index . #t) + (db/doc . "Label country ISO code")) + + `((db/ident . label/startYear) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Year label was founded")) + + `((db/ident . label/endYear) + (db/valueType . db.type/long) + (db/cardinality . db.cardinality/one) + (db/doc . "Year label ceased operations")) + + ;; ---- Release-Label join ---- + `((db/ident . release/labels) + (db/valueType . db.type/ref) + (db/cardinality . db.cardinality/many) + (db/doc . "Labels associated with this release")))) + +;; ---- Installer ---- + +(def (install-mbrainz-schema! conn) + (displayln "Installing MBrainz schema...") + (let ([report (transact! conn (mbrainz-schema-ops))]) + (let ([n (length (tx-report-tx-data report))]) + (displayln " Installed " (quotient n 3) " attributes (" n " datoms)")) + report)) + +;; ---- If run as script ---- + +(def (main) + (let ([conn (connect ":memory:")]) + (install-mbrainz-schema! conn) + (let ([attrs (schema-for conn)]) + (displayln " Schema attributes: " (length attrs)) + (displayln " Sample: " (map db-attribute-ident (take attrs 5)))))) + +(main) --- a/jerboa-db.md +++ b/jerboa-db.md @@ -4,16 +4,17 @@ index storage and DuckDB for analytics. Single binary, embeddable, distributed, with a Datalog query engine and immutable time-travel over all data. -**Status:** 2026-04-12 — **Phases 1–8 implemented. All 34 integration tests -pass.** Implementation lives in `lib/jerboa-db/` (Jerboa library files using +**Status:** 2026-04-13 — **Phases 1–8 implemented. All 34 integration tests +pass. MBrainz benchmark harness complete and running (all 8 queries verified).** Implementation lives in `lib/jerboa-db/` (Jerboa library files using