perf: Phase 2.3 -- auto-route eligible queries to DuckDB via parameter
ober
a48508526266ec044812781f29b9c35e80bb35cb
--- a/benchmarks/mbrainz-bench.ss +++ b/benchmarks/mbrainz-bench.ss @@ -341,6 +341,10 @@ (lambda () (q4 db 240000))) (bench "Q4*" " via DuckDB fallback" (lambda () (q4-analytics db ae 240000))) + (bench "Q4↻" " auto-routed (current-analytics-engine)" + (lambda () + (parameterize ([current-analytics-engine ae]) + (q4 db 240000)))) (bench "Q5" "Count releases per country" (lambda () (q5 db))) (bench "Q6" "All releases for one artist (reverse ref)" @@ -350,7 +354,11 @@ (bench "Q8" "Avg track duration by release status" (lambda () (q8 db))) (bench "Q8*" " via DuckDB fallback" - (lambda () (q8-analytics db ae))))]) + (lambda () (q8-analytics db ae))) + (bench "Q8↻" " auto-routed (current-analytics-engine)" + (lambda () + (parameterize ([current-analytics-engine ae]) + (q8 db)))))]) (displayln (make-string 80 #\-)) (let ([total (apply + (map cdr results))]) --- a/lib/jerboa-db/query/engine.ss +++ b/lib/jerboa-db/query/engine.ss @@ -8,7 +8,10 @@ (library (jerboa-db query engine) (export query-db parse-query explain-query parsed-query? parsed-query-find-vars parsed-query-in-vars - parsed-query-where-clauses parsed-query-rules) + parsed-query-where-clauses parsed-query-rules + ;; Auto-routing hooks (analytical fallback wires through these) + current-analytics-engine + set-analytical-query-handler!) (import (except (chezscheme) make-hash-table hash-table? @@ -997,6 +1000,50 @@ '() (evaluate-plan db rest new-bindings schema rules-ht))))) + ;; ---- Auto-routing to DuckDB analytical engine ---- + ;; + ;; `current-analytics-engine` is a parameter holding either #f (no engine) + ;; or an analytics-engine value. When set, query-db will route eligible + ;; queries (multi-clause joins, aggregations) through SQL. + ;; + ;; sql-translate.ss installs the actual handler at load time via + ;; `set-analytical-query-handler!`; this avoids a circular import + ;; (sql-translate already imports engine). + + (def current-analytics-engine (make-parameter #f)) + (def *analytical-query-handler* #f) + + (def (set-analytical-query-handler! proc) + (set! *analytical-query-handler* proc)) + + ;; Should we attempt the SQL fallback? Only when: + ;; - an analytics engine is bound, + ;; - a handler has been registered, + ;; - the query has either ≥3 data clauses, or aggregation + ≥2 data clauses. + ;; Small queries (Q1/Q2) hit Datalog in <1ms; DuckDB round-trip is ~5ms. + (def (analytics-route-eligible? find-vars where-clauses) + (and *analytical-query-handler* + (current-analytics-engine) + (let* ([data-clauses (filter (lambda (c) + (and (pair? c) + (not (pair? (car c))) + (>= (length c) 3) + (symbol? (cadr c)) + (let ([s (symbol->string (cadr c))]) + (or (zero? (string-length s)) + (not (char=? (string-ref s 0) #\?)))))) + where-clauses)] + [n (length data-clauses)] + [has-agg? (exists (lambda (fv) + (and (pair? fv) + (memq (car fv) + '(count sum avg min max + count-distinct median + variance stddev variance-sample)))) + find-vars)]) + (or (>= n 3) + (and has-agg? (>= n 2)))))) + ;; ---- Top-level query function ---- (def (query-db parsed db . inputs) @@ -1006,10 +1053,19 @@ [where-clauses (parsed-query-where-clauses parsed)] [rules-ht (parsed-query-rules parsed)] [pure-agg-desc (pure-aggregate-eligible? find-vars in-vars where-clauses)]) - (if pure-agg-desc - (execute-pure-aggregate db pure-agg-desc) - (query-db-general schema find-vars in-vars where-clauses rules-ht - db inputs)))) + (cond + [pure-agg-desc + (execute-pure-aggregate db pure-agg-desc)] + [(analytics-route-eligible? find-vars where-clauses) + (let ([rows (apply *analytical-query-handler* + parsed (current-analytics-engine) schema inputs)]) + (if rows + rows + (query-db-general schema find-vars in-vars where-clauses rules-ht + db inputs)))] + [else + (query-db-general schema find-vars in-vars where-clauses rules-ht + db inputs)]))) (def (query-db-general schema find-vars in-vars where-clauses rules-ht db inputs) (let* ([rules-ht rules-ht] --- a/lib/jerboa-db/query/sql-translate.ss +++ b/lib/jerboa-db/query/sql-translate.ss @@ -392,4 +392,7 @@ [(= n 0) (cdar lst)] [else (cdr-of-nth-pair (cdr lst) (- n 1))])) + ;; Register with engine.ss so query-db can auto-route eligible queries. + (set-analytical-query-handler! analytical-query) + ) ;; end library