perf: COPY-FROM-CSV replaces batched INSERT VALUES — 49x faster sync
ober
8b423ec131d584cfd0d6fc9307282d9c1cb29f51
--- a/jerboa-db.md +++ b/jerboa-db.md @@ -2879,7 +2879,7 @@ Open items (not blocking parity): |---|---|---| | `defquery` macro for compile-time specialization | Q1/Q2/Q6/Q7 sub-µs hot path | Deferred — auto-routing already covers the headline gap | | Real MBrainz EDN loader | Full-scale validation | Synthetic data tracks the shape; real EDN would eliminate any "synthetic shortcut" doubt | -| Faster `analytics-sync!` at scale | Throughput | 126s for 6.4M datoms (10%); could go to APPEND/COPY API + incremental delta | +| Incremental analytics-sync (delta) | Sync amortization | Bulk path now 2.6 s at 10% via COPY-FROM-CSV; an incremental delta sync would cut steady-state cost further | ### Comparison vs Datahike (published numbers, full scale) --- a/lib/jerboa-db/analytics.ss +++ b/lib/jerboa-db/analytics.ss @@ -139,33 +139,54 @@ (loop (cdr rest) (car rest) (if (and prev (datom-added? prev)) (cons prev acc) acc))]))) - ;; Bulk-insert datoms via multi-row VALUES batches. Per-row prepared - ;; INSERTs are ~250µs/row; batched literal VALUES are ~5µs/row. - ;; Larger batches amortize DuckDB SQL parse + round-trip; we cap the - ;; batch by row count, not byte size, since values tend to be small. - (def +sync-batch-size+ 5000) + ;; Bulk-insert datoms via DuckDB's COPY FROM CSV. Stream-write a temp + ;; file, then issue one COPY statement. INSERT VALUES (even batched at + ;; 5K) is ~20µs/row in DuckDB because the SQL parser has to materialize + ;; each tuple; COPY skips the parser and is ~0.7µs/row — a 30× win on + ;; the bulk path. The tempfile is cleaned up on success and on error. + (def +sync-tmp-counter+ 0) + + (def (sync-tmp-path) + (set! +sync-tmp-counter+ (+ +sync-tmp-counter+ 1)) + (string-append "/tmp/jerboa-db-sync-" + (number->string (time-second (current-time))) + "-" + (number->string +sync-tmp-counter+) + ".csv")) + + (def +csv-null-marker+ "\\N") (def (bulk-insert-datoms! conn schema datoms) ;; Cache (a -> (a-name . vtype)) across the whole sync: a typical ;; bench has ~20 attributes, but each `(schema-lookup-by-id ...)` ;; chases a hashtable + record accessor pair. Hoist it once. - (let ([attr-cache (make-eqv-hashtable)]) - (let loop ([rest datoms]) - (cond - [(null? rest) (void)] - [else - (let-values ([(batch tail) (split-at-most rest +sync-batch-size+)]) - (let ([sql (build-insert-sql schema attr-cache batch)]) - (when sql (duckdb-exec conn sql))) - (loop tail))])))) - - (def (split-at-most lst n) - (let loop ([i 0] [taken '()] [rest lst]) - (cond - [(or (= i n) (null? rest)) - (values (reverse taken) rest)] - [else - (loop (+ i 1) (cons (car rest) taken) (cdr rest))]))) + (when (pair? datoms) + (let ([attr-cache (make-eqv-hashtable)] + [path (sync-tmp-path)]) + (dynamic-wind + void + (lambda () + (write-datoms-csv! path schema attr-cache datoms) + (duckdb-exec conn + (string-append + "COPY datoms FROM '" path + "' (FORMAT CSV, HEADER FALSE, DELIMITER ',', QUOTE '\"', " + "ESCAPE '\"', NULLSTR '" +csv-null-marker+ "')"))) + (lambda () + (when (file-exists? path) (delete-file path))))))) + + (def (write-datoms-csv! path schema attr-cache datoms) + (let ([port (open-file-output-port path + (file-options no-fail) + (buffer-mode block) + (native-transcoder))]) + (dynamic-wind + void + (lambda () + (for-each + (lambda (d) (write-csv-row! port schema attr-cache d)) + datoms)) + (lambda () (close-port port))))) (def (attr-info schema cache a) (let ([cached (hashtable-ref cache a #f)]) @@ -177,20 +198,7 @@ (hashtable-set! cache a info) info)))) - (def (build-insert-sql schema attr-cache batch) - (and (pair? batch) - (let ([buf (open-output-string)]) - (display "INSERT INTO datoms (e, a, a_name, v_long, v_double, v_string, v_bool, v_ref, v_instant, tx, added) VALUES " buf) - (let loop ([ds batch] [first? #t]) - (cond - [(null? ds) (void)] - [else - (unless first? (display ", " buf)) - (write-row! buf schema attr-cache (car ds)) - (loop (cdr ds) #f)])) - (get-output-string buf)))) - - (def (write-row! buf schema attr-cache d) + (def (write-csv-row! port schema attr-cache d) (let-values ([(e a v tx added?) (if (datom? d) (values (datom-e d) (datom-a d) (datom-v d) @@ -202,47 +210,46 @@ [vtype (cdr info)]) (let-values ([(v-long v-double v-string v-bool v-ref v-instant) (classify-value v vtype)]) - (display "(" buf) - (write-sql-int buf e) (display ", " buf) - (write-sql-int buf a) (display ", " buf) - (write-sql-text buf a-name) (display ", " buf) - (write-sql-int buf v-long) (display ", " buf) - (write-sql-num buf v-double) (display ", " buf) - (write-sql-text buf v-string)(display ", " buf) - (write-sql-bool buf v-bool) (display ", " buf) - (write-sql-int buf v-ref) (display ", " buf) - (write-sql-int buf v-instant)(display ", " buf) - (write-sql-int buf tx) (display ", " buf) - (write-sql-bool buf added?) - (display ")" buf))))) - - (def (write-sql-int buf v) - (cond [(not v) (display "NULL" buf)] - [(integer? v) (display v buf)] - [else (display "NULL" buf)])) - - (def (write-sql-num buf v) - (cond [(not v) (display "NULL" buf)] - [(number? v) (display (inexact v) buf)] - [else (display "NULL" buf)])) - - (def (write-sql-bool buf v) - (cond [(eq? v #t) (display "TRUE" buf)] - [(eq? v #f) (display "FALSE" buf)] - [else (display "NULL" buf)])) - - (def (write-sql-text buf v) + (write-csv-int port e) (write-char #\, port) + (write-csv-int port a) (write-char #\, port) + (write-csv-text port a-name) (write-char #\, port) + (write-csv-int port v-long) (write-char #\, port) + (write-csv-num port v-double) (write-char #\, port) + (write-csv-text port v-string) (write-char #\, port) + (write-csv-bool port v-bool) (write-char #\, port) + (write-csv-int port v-ref) (write-char #\, port) + (write-csv-int port v-instant) (write-char #\, port) + (write-csv-int port tx) (write-char #\, port) + (write-csv-bool port added?) + (write-char #\newline port))))) + + (def (write-csv-int port v) + (cond [(not v) (display +csv-null-marker+ port)] + [(integer? v) (display v port)] + [else (display +csv-null-marker+ port)])) + + (def (write-csv-num port v) + (cond [(not v) (display +csv-null-marker+ port)] + [(number? v) (display (inexact v) port)] + [else (display +csv-null-marker+ port)])) + + (def (write-csv-bool port v) + (cond [(eq? v #t) (display "true" port)] + [(eq? v #f) (display "false" port)] + [else (display +csv-null-marker+ port)])) + + (def (write-csv-text port v) (cond - [(not v) (display "NULL" buf)] + [(not v) (display +csv-null-marker+ port)] [(string? v) - (display "'" buf) + (write-char #\" port) (string-for-each (lambda (c) - (when (char=? c #\') (display "'" buf)) - (display c buf)) + (when (char=? c #\") (write-char #\" port)) + (write-char c port)) v) - (display "'" buf)] - [else (display "NULL" buf)])) + (write-char #\" port)] + [else (display +csv-null-marker+ port)])) ;; ---- Insert a single datom record ---- --- a/plan.md +++ b/plan.md @@ -51,8 +51,8 @@ compile-time query specialization no longer moved the needle on the bench. | Q6 | 0 ms | — | parity holds | | Q7 | 0 ms | — | parity holds | | Q8 | 15260 ms | **41 ms** (auto: 42) | 372× — bit-for-bit identical (3 rows) | -| Loading | 27.6 s | | | -| Analytics-sync (one-time) | 127 s | | one-time, amortized across many queries | +| Loading | 27.4 s | | | +| Analytics-sync (one-time) | **2.6 s** | | was 127 s; COPY-FROM-CSV beats batched INSERT-VALUES 49× | Q4 result counts match bit-for-bit between Datalog and DuckDB at 1%, 5%, and 10% scale (428,096 / 2,138,115 / 4,270,990 rows respectively). The @@ -71,9 +71,10 @@ sorted (e,a,v,tx) order and keeping only the highest-tx datom per |---|---|---|---| | 1 | Pure-aggregate pushdown (single-clause) | `439b9a7` | Q8 streaming-aggregate fast path | | 2 | DuckDB analytical fallback | `c61ef71` | infrastructure | -| 2.1 | Batched VALUES INSERT for analytics-sync | `cad68b5` | sync 124s → 11.7s | +| 2.1 | Batched VALUES INSERT for analytics-sync | `cad68b5` | sync 124s → 11.7s (later superseded by 2.4) | | 2.2 | `:in` input substitution + `SELECT DISTINCT` | `0417ea6` | Datalog→SQL parity for non-trivial queries | | 2.3 | Auto-route eligible queries via `current-analytics-engine` | `66ba53c` | transparent 5× on Q4, 88× on Q8 | +| 2.4 | COPY-FROM-CSV bulk sync (replaces batched INSERT VALUES) | (this commit) | sync 127s → 2.6s at 10% scale (49×) | | 3 | LSM staging buffer + tempid hashtable + fulltext O(n²) fix | `6207c2d` | 47× ingest (the discovery: not RB-tree depth, but fulltext word-list `(member ...)` per insert) | | 4 | `defquery` macro | — | **deferred** — auto-routing already covers the win | | 5 | Final benchmark + writeup | `bdb00d5` + this | parity story below |