query/planner: order value-bound clauses by real per-(attr,value) count
ober
d685a8f9509692362df6971ad199d50daed4582d
--- a/lib/jerboa-db/query/engine.ss +++ b/lib/jerboa-db/query/engine.ss @@ -33,6 +33,17 @@ (jerboa-db query planner) (jerboa-db query rules)) + ;; ---- Cardinality estimator (planner input) ---- + ;; (card-fn aid value) -> count of current datoms with that attribute id and + ;; value, from the live AVET index using the B+-tree's O(log n) range count. + ;; Lets the planner order value-bound clauses by actual value frequency. + (def (make-card-fn db) + (let ([avet (db-resolve-index db 'avet)]) + (lambda (aid v) + (dbi-count avet + (make-datom 0 aid v 0 #t) + (make-datom (greatest-fixnum) aid v (greatest-fixnum) #t))))) + ;; ---- Logic variable helpers ---- (def (logic-var? x) @@ -1145,7 +1156,8 @@ [bound-at-start (if (null? input-bindings) '() (vector->list (hashtable-keys (car input-bindings))))] [db-stats (db-value-stats db)] - [ordered-clauses (reorder-clauses where-clauses bound-at-start schema db-stats)] + [card-fn (make-card-fn db)] + [ordered-clauses (reorder-clauses where-clauses bound-at-start schema db-stats card-fn)] ;; Compute live variable sets for projection pushdown [live-vars-steps (compute-live-vars-at-each-step ordered-clauses find-vars)] ;; Compute hash-join plan for top-level clause execution @@ -1197,7 +1209,8 @@ [else (loop (cdr ivs) (cdr inps) (cons (car ivs) bound))]))] [db-stats (db-value-stats db-val)] - [ordered-clauses (reorder-clauses where-clauses bound-at-start schema db-stats)]) + [card-fn (make-card-fn db-val)] + [ordered-clauses (reorder-clauses where-clauses bound-at-start schema db-stats card-fn)]) ;; Build plan: for each clause, describe it and the chosen index (let build-plan ([clauses ordered-clauses] [bound bound-at-start] [plan '()]) (if (null? clauses) --- a/lib/jerboa-db/query/planner.ss +++ b/lib/jerboa-db/query/planner.ss @@ -97,8 +97,20 @@ ;; Higher score = more selective = should come first. ;; Optional db-stats argument enables attribute-count-based scoring. + ;; Map a per-(attribute,value) match count to a value-bound clause score + ;; (52-95): a near-unique value scores ~95 (effectively a point lookup), a + ;; low-selectivity value (matches many rows) scores ~52. Monotone in cnt. + (def (value-card-score cnt) + (cond + [(<= cnt 1) 95] + [else (max 52 (- 92 (* 3 (integer-length cnt))))])) + + ;; rest = (db-stats card-fn). card-fn, when supplied, is (card-fn aid value) + ;; -> count of current datoms with that attribute id and value (from the live + ;; AVET index, O(log n)); it refines value-bound clause selectivity. (def (score-clause clause bound-vars schema . rest) - (let ([db-stats (and (pair? rest) (car rest))]) + (let ([db-stats (and (pair? rest) (car rest))] + [card-fn (and (pair? rest) (pair? (cdr rest)) (cadr rest))]) (cond ;; Not clause: should come after its sub-clause vars are bound. ;; Score low so it's placed late (it's a filter). @@ -111,7 +123,7 @@ (if (for-all (lambda (v) (memq v bound-vars)) join-vars) 3 -100))] ;; Or clause: score like the best alternative [(and (pair? clause) (eq? (car clause) 'or)) - (let ([alt-scores (map (lambda (alt) (score-clause alt bound-vars schema db-stats)) + (let ([alt-scores (map (lambda (alt) (score-clause alt bound-vars schema db-stats card-fn)) (cdr clause))]) (if (null? alt-scores) 0 (apply max alt-scores)))] ;; Data pattern: (?e attr ?v ...) @@ -122,25 +134,31 @@ [v-pos (caddr clause)] [e-bound? (or (not (logic-var? e-pos)) (memq e-pos bound-vars))] [a-bound? (not (logic-var? a-pos))] ;; attrs are always concrete - [v-bound? (or (not (logic-var? v-pos)) (memq v-pos bound-vars))]) - (let* ([attr (and a-bound? (schema-lookup-by-ident schema a-pos))] - [aid (and attr (db-attribute-id attr))] - ;; Stats-based bonus: smaller attribute -> higher score. - ;; Only applies when entity is unbound (otherwise EAVT point-lookup - ;; dominates regardless of attribute cardinality). - ;; Divide by 10: contributes 0-90, below entity-bound (100) and - ;; unique-attr (90) but above plain attribute-bound (20). - [stat-bonus (if (and db-stats aid (not e-bound?)) - (quotient (db-stats-selectivity-score db-stats aid) 10) - 0)]) - (+ - (if e-bound? 100 0) ;; entity bound: very selective - (if (and attr (db-attribute-unique attr)) 90 0) ;; unique attr: point lookup - ;; V-bound + scalar attr: AVET range scan (fast). - ;; V-bound + ref attr: VAET reverse lookup (also fast). - (if v-bound? (if (and attr (avet-eligible? attr)) 60 50) 0) - (if a-bound? 20 0) - stat-bonus)))] ;; 0-90 based on attribute selectivity + [v-bound? (or (not (logic-var? v-pos)) (memq v-pos bound-vars))] + [concrete-v? (not (logic-var? v-pos))] + [attr (and a-bound? (schema-lookup-by-ident schema a-pos))] + [aid (and attr (db-attribute-id attr))] + ;; Precise per-(attribute,value) cardinality: only when the value + ;; is a concrete literal, the entity is unbound, and the attr is + ;; AVET-indexed (so we can actually count it). This is strictly + ;; better selectivity info than the per-attribute heuristic. + [card (and card-fn aid (not e-bound?) concrete-v? + attr (avet-eligible? attr) + (card-fn aid v-pos))]) + (cond + ;; entity bound: EAVT point lookup + [e-bound? (+ 100 (if a-bound? 20 0))] + ;; concrete value with a real count: order by actual frequency + [card (+ (value-card-score card) (if a-bound? 20 0))] + ;; unique attribute: effectively a point lookup + [(and attr (db-attribute-unique attr)) (+ 90 (if a-bound? 20 0))] + ;; fall back to the value-bound heuristic + per-attribute stat bonus + [else + (+ (if v-bound? (if (and attr (avet-eligible? attr)) 60 50) 0) + (if a-bound? 20 0) + (if (and db-stats aid) + (quotient (db-stats-selectivity-score db-stats aid) 10) + 0))]))] ;; Predicate/function clauses: if ALL vars are bound, score maximally so ;; the clause fires as an early filter immediately after its last dep. ;; Otherwise score proportionally to bound vars. @@ -157,14 +175,15 @@ ;; Optional db-stats argument enables attribute-count-based ordering. (def (reorder-clauses clauses initial-bound-vars schema . rest) - (let ([db-stats (and (pair? rest) (car rest))]) + (let ([db-stats (and (pair? rest) (car rest))] + [card-fn (and (pair? rest) (pair? (cdr rest)) (cadr rest))]) (let loop ([remaining clauses] [bound-vars initial-bound-vars] [result '()]) (if (null? remaining) (reverse result) (let* ([scored (map (lambda (c) - (cons (score-clause c bound-vars schema db-stats) c)) + (cons (score-clause c bound-vars schema db-stats card-fn) c)) remaining)] [sorted (sort scored (lambda (a b) (> (car a) (car b))))] [best (cdar sorted)]