fix: analytics-sync now filters to currently-live datoms
ober
aba0a19d6547881346a6441e1950ce268748526f
--- a/lib/jerboa-db/analytics.ss +++ b/lib/jerboa-db/analytics.ss @@ -87,17 +87,20 @@ ;; ---- Sync from a db-value snapshot ---- ;; - ;; Reads all datoms from the EAVT index and (re)loads them into DuckDB. - ;; For simplicity this does a full reload each time (truncate + insert). - ;; An incremental variant would track last-synced-tx and only insert new - ;; datoms — feasible but requires filtering by tx, so we keep full reload - ;; for correctness. + ;; Reads all datoms from the EAVT index, filters to currently-live ones, + ;; and (re)loads them into DuckDB. Filtering is required because EAVT + ;; preserves retractions and superseded values, but SQL queries against + ;; the analytics replica use `WHERE added` to filter retractions — that + ;; filter would still match the original assertion's row. Streaming + ;; through EAVT-sorted (e,a,v,tx) order lets us keep only the latest + ;; datom per (e,a,v), then drop those whose latest state is retraction. (def (analytics-sync! ae db-val) (let* ([schema (analytics-engine-schema-ref ae)] [indices (db-value-indices db-val)] [eavt (index-set-eavt indices)] [datoms (dbi-datoms eavt)] + [live (live-datoms datoms)] [conn (analytics-engine-duckdb-conn ae)]) (duckdb-exec conn "DELETE FROM datoms") (duckdb-exec conn "DELETE FROM attrs") @@ -114,10 +117,28 @@ (let ([c (db-attribute-cardinality attr)]) (if c (symbol->string c) #f)))) (schema-all-attributes schema)) - (bulk-insert-datoms! conn schema datoms) + (bulk-insert-datoms! conn schema live) (analytics-engine-last-synced-tx-set! ae (db-value-basis-tx db-val)) (analytics-engine-synced?-set! ae #t))) + ;; Stream EAVT-sorted datoms, collapse each (e,a,v) run to its highest-tx + ;; entry, and drop entries whose final state is a retraction. The input + ;; is sorted by (e,a,v,tx) so equal-(e,a,v) datoms are contiguous and the + ;; last one wins. + (def (live-datoms eavt-datoms) + (let loop ([rest eavt-datoms] [prev #f] [acc '()]) + (cond + [(null? rest) + (reverse (if (and prev (datom-added? prev)) (cons prev acc) acc))] + [(and prev + (eqv? (datom-e (car rest)) (datom-e prev)) + (eqv? (datom-a (car rest)) (datom-a prev)) + (equal? (datom-v (car rest)) (datom-v prev))) + (loop (cdr rest) (car rest) acc)] + [else + (loop (cdr rest) (car rest) + (if (and prev (datom-added? prev)) (cons prev acc) acc))]))) + ;; Bulk-insert datoms via multi-row VALUES batches. Per-row prepared ;; INSERTs are ~250µs/row; batched literal VALUES are ~5µs/row. (def +sync-batch-size+ 1000) --- a/plan.md +++ b/plan.md @@ -39,8 +39,31 @@ compile-time query specialization no longer moved the needle on the bench. | Q4 (2.14M result rows) | 10018 ms | **1467 ms** | 6.8× | | Q8 | 6407 ms | **31 ms** | 207× | -Q4 result counts match bit-for-bit between Datalog and DuckDB at 1% and 5% -scale (428,096 and 2,138,115 rows respectively). +### MBrainz, 10% scale (26,200 artists / 131,000 releases / 1,310,000 tracks) + +| Query | Datalog | DuckDB fallback | Notes | +|---|---:|---:|---| +| Q1 | 0 ms | — | parity holds | +| Q2 | 8 ms | — | parity holds | +| Q3 | 9 ms | — | parity holds | +| Q4 (4.27M result rows) | 20451 ms | **6476 ms** | bit-for-bit identical after retraction-aware sync fix | +| Q5 | 35 ms | — | parity holds | +| Q6 | 0 ms | — | parity holds | +| Q7 | 0 ms | — | parity holds | +| Q8 | 15800 ms | **45 ms** | 351× — bit-for-bit identical (3 rows) | +| Loading | 25.6 s | | | +| Analytics-sync (one-time) | 126 s | | | + +Q4 result counts match bit-for-bit between Datalog and DuckDB at 1%, 5%, +and 10% scale (428,096 / 2,138,115 / 4,270,990 rows respectively). The +10%-scale divergence first observed during Phase 5 was traced to +`analytics-sync!` indiscriminately copying every EAVT entry into DuckDB, +including assertions that had been later retracted — `WHERE added` still +matched the original assertion's row. Triggered in the bench by +`make-uuid!`'s 8192-cycle period (lower-16-bit LCG drift), which causes +gid collisions and cardinality-one upserts. Fixed by streaming EAVT in +sorted (e,a,v,tx) order and keeping only the highest-tx datom per +(e,a,v), then dropping retracted ones. ### Phases as shipped