perf: 5x sync batch + per-attr schema-lookup memoization
ober
f740e1caaa524911455848452283d3fdd4e69bbe
--- a/lib/jerboa-db/analytics.ss +++ b/lib/jerboa-db/analytics.ss @@ -141,17 +141,23 @@ ;; Bulk-insert datoms via multi-row VALUES batches. Per-row prepared ;; INSERTs are ~250µs/row; batched literal VALUES are ~5µs/row. - (def +sync-batch-size+ 1000) + ;; Larger batches amortize DuckDB SQL parse + round-trip; we cap the + ;; batch by row count, not byte size, since values tend to be small. + (def +sync-batch-size+ 5000) (def (bulk-insert-datoms! conn schema datoms) - (let loop ([rest datoms]) - (cond - [(null? rest) (void)] - [else - (let-values ([(batch tail) (split-at-most rest +sync-batch-size+)]) - (let ([sql (build-insert-sql schema batch)]) - (when sql (duckdb-exec conn sql))) - (loop tail))]))) + ;; Cache (a -> (a-name . vtype)) across the whole sync: a typical + ;; bench has ~20 attributes, but each `(schema-lookup-by-id ...)` + ;; chases a hashtable + record accessor pair. Hoist it once. + (let ([attr-cache (make-eqv-hashtable)]) + (let loop ([rest datoms]) + (cond + [(null? rest) (void)] + [else + (let-values ([(batch tail) (split-at-most rest +sync-batch-size+)]) + (let ([sql (build-insert-sql schema attr-cache batch)]) + (when sql (duckdb-exec conn sql))) + (loop tail))])))) (def (split-at-most lst n) (let loop ([i 0] [taken '()] [rest lst]) @@ -161,7 +167,17 @@ [else (loop (+ i 1) (cons (car rest) taken) (cdr rest))]))) - (def (build-insert-sql schema batch) + (def (attr-info schema cache a) + (let ([cached (hashtable-ref cache a #f)]) + (or cached + (let* ([attr (schema-lookup-by-id schema a)] + [a-name (and attr (symbol->string (db-attribute-ident attr)))] + [vtype (and attr (db-attribute-value-type attr))] + [info (cons a-name vtype)]) + (hashtable-set! cache a info) + info)))) + + (def (build-insert-sql schema attr-cache batch) (and (pair? batch) (let ([buf (open-output-string)]) (display "INSERT INTO datoms (e, a, a_name, v_long, v_double, v_string, v_bool, v_ref, v_instant, tx, added) VALUES " buf) @@ -170,20 +186,20 @@ [(null? ds) (void)] [else (unless first? (display ", " buf)) - (write-row! buf schema (car ds)) + (write-row! buf schema attr-cache (car ds)) (loop (cdr ds) #f)])) (get-output-string buf)))) - (def (write-row! buf schema d) + (def (write-row! buf schema attr-cache d) (let-values ([(e a v tx added?) (if (datom? d) (values (datom-e d) (datom-a d) (datom-v d) (datom-tx d) (datom-added? d)) (values (vector-ref d 0) (vector-ref d 1) (vector-ref d 2) (vector-ref d 3) (vector-ref d 4)))]) - (let* ([attr (schema-lookup-by-id schema a)] - [a-name (and attr (symbol->string (db-attribute-ident attr)))] - [vtype (and attr (db-attribute-value-type attr))]) + (let* ([info (attr-info schema attr-cache a)] + [a-name (car info)] + [vtype (cdr info)]) (let-values ([(v-long v-double v-string v-bool v-ref v-instant) (classify-value v vtype)]) (display "(" buf)