Vendor jsqlite for std sqlite
ober
021d0bc131621e0b8937bbe7a3d8a60e0feb6af4
--- a/Makefile +++ b/Makefile @@ -33,13 +33,13 @@ CHEZ_INSTALL_FLAGS = \ --installman=$(CHEZ_PREFIX)/share/man \ --installdoc=$(CHEZ_PREFIX)/share/doc \ --as-is -LIBDIRS = lib +LIBDIRS = lib:$(JERBOA_HOME)/vendor/jsqlite/src # Base directory for chez-* repos (legacy C FFI — see `make native` for Rust backend) JERBOA_EXT_DIR ?= $(HOME)/src # External chez-* library paths for legacy wrapper modules -JERBOA_EXT_LIBDIRS = $(JERBOA_EXT_DIR)/jerboa-https/src:$(JERBOA_EXT_DIR)/jerboa-ssl/src:$(JERBOA_EXT_DIR)/chez-zlib/src:$(JERBOA_EXT_DIR)/jerboa-pcre2:$(JERBOA_EXT_DIR)/chez-leveldb:$(JERBOA_EXT_DIR)/chez-epoll/src:$(JERBOA_EXT_DIR)/jerboa-inotify/src:$(JERBOA_EXT_DIR)/jerboa-crypto/src:$(JERBOA_EXT_DIR)/jerboa-sqlite/src:$(JERBOA_EXT_DIR)/chez-postgresql/src +JERBOA_EXT_LIBDIRS = $(JERBOA_EXT_DIR)/jerboa-https/src:$(JERBOA_EXT_DIR)/jerboa-ssl/src:$(JERBOA_EXT_DIR)/chez-zlib/src:$(JERBOA_EXT_DIR)/jerboa-pcre2:$(JERBOA_EXT_DIR)/chez-leveldb:$(JERBOA_EXT_DIR)/chez-epoll/src:$(JERBOA_EXT_DIR)/jerboa-inotify/src:$(JERBOA_EXT_DIR)/jerboa-crypto/src:$(JERBOA_EXT_DIR)/chez-postgresql/src # Shared object paths for legacy FFI-based chez-* libraries -JERBOA_EXT_LDPATH = $(JERBOA_EXT_DIR)/jerboa-ssl:$(JERBOA_EXT_DIR)/chez-zlib:$(JERBOA_EXT_DIR)/jerboa-pcre2:$(JERBOA_EXT_DIR)/chez-leveldb:$(JERBOA_EXT_DIR)/chez-epoll:$(JERBOA_EXT_DIR)/jerboa-inotify:$(JERBOA_EXT_DIR)/jerboa-crypto:$(JERBOA_EXT_DIR)/jerboa-sqlite:$(JERBOA_EXT_DIR)/chez-postgresql +JERBOA_EXT_LDPATH = $(JERBOA_EXT_DIR)/jerboa-ssl:$(JERBOA_EXT_DIR)/chez-zlib:$(JERBOA_EXT_DIR)/jerboa-pcre2:$(JERBOA_EXT_DIR)/chez-leveldb:$(JERBOA_EXT_DIR)/chez-epoll:$(JERBOA_EXT_DIR)/jerboa-inotify:$(JERBOA_EXT_DIR)/jerboa-crypto:$(JERBOA_EXT_DIR)/chez-postgresql PURE_AUDIT_ROOT ?= $(HOME)/mine PURE_AUDIT_ARGS ?= --summary --discover $(PURE_AUDIT_ROOT) TYPED_SOURCES ?= tests/fixtures/typed/valid-split-tree.ss @@ -1240,7 +1240,7 @@ test-wrappers: || echo " crypto: SKIP (requires jerboa_crypto_shim.so)" @LD_LIBRARY_PATH="$(JERBOA_EXT_LDPATH):$$LD_LIBRARY_PATH" \ $(SCHEME) --libdirs "$(LIBDIRS):$(JERBOA_EXT_LIBDIRS)" --script tests/test-wrapper-sqlite.ss 2>/dev/null \ - || echo " sqlite: SKIP (requires jerboa_sqlite_shim.so)" + || echo " sqlite: FAIL (jsqlite wrapper test failed)" @LD_LIBRARY_PATH="$(JERBOA_EXT_LDPATH):$$LD_LIBRARY_PATH" \ $(SCHEME) --libdirs "$(LIBDIRS):$(JERBOA_EXT_LIBDIRS)" --script tests/test-wrapper-postgresql.ss 2>/dev/null \ || echo " postgresql: SKIP (requires jerboa_pg_shim.so)" --- a/data/api-signatures.sexp +++ b/data/api-signatures.sexp @@ -1,4 +1,4 @@ -(("errors") ("generated" . "2026-06-11") +(("errors") ("generated" . "2026-06-14") ("modules" ("(jerboa build musl)" ("exports" "build-musl-binary" "make-musl-cross-target" @@ -2349,14 +2349,20 @@ ("(std db sqlite)" ("exports" "SQLITE_BLOB" "SQLITE_DONE" "SQLITE_FLOAT" "SQLITE_INTEGER" "SQLITE_NULL" "SQLITE_OK" "SQLITE_ROW" - "SQLITE_TEXT" "sqlite-bind!" "sqlite-bind-null!" - "sqlite-changes" "sqlite-clear-bindings" "sqlite-close" - "sqlite-column-count" "sqlite-column-name" + "SQLITE_TEXT" "sqlite-bind!" "sqlite-bind-blob!" + "sqlite-bind-int!" "sqlite-bind-null!" + "sqlite-bind-parameter-count" "sqlite-bind-parameter-index" + "sqlite-bind-parameter-name" "sqlite-bind-real!" + "sqlite-bind-text!" "sqlite-changes" "sqlite-clear-bindings" + "sqlite-close" "sqlite-column-count" "sqlite-column-name" "sqlite-column-type" "sqlite-column-value" "sqlite-columns" - "sqlite-errmsg" "sqlite-eval" "sqlite-exec" - "sqlite-finalize" "sqlite-last-insert-rowid" "sqlite-open" - "sqlite-prepare" "sqlite-query" "sqlite-reset" - "sqlite-step") + "sqlite-db->bytevector" "sqlite-db?" "sqlite-errcode" + "sqlite-errmsg" "sqlite-eval" "sqlite-exec" "sqlite-execute" + "sqlite-extended-errcode" "sqlite-finalize" + "sqlite-last-insert-rowid" "sqlite-open" + "sqlite-open-bytevector" "sqlite-open-v2" "sqlite-prepare" + "sqlite-query" "sqlite-reset" "sqlite-row" "sqlite-step" + "sqlite-stmt?" "sqlite-total-changes") ("file" . "lib/std/db/sqlite.ss") ("tier" . "stable")) ("(std db sqlite-native)" @@ -6930,13 +6936,13 @@ ("file" . "lib/std/zipper.ss") ("tier" . "stable"))) ("source_root" . "/Users/user/mine/jerboa") - ("stats" ("modules" . 707) ("parse_errors" . 0) ("symbols" . 10251) + ("stats" ("modules" . 707) ("parse_errors" . 0) ("symbols" . 10267) ("tiers" ("compat" . 47) ("core" . 34) ("stable" . 548) ("unstable" . 78)) - ("total_exports" . 13449)) + ("total_exports" . 13466)) ("symbol_index" ("%chan-enqueue-raw!" "(std csp)") ("&actor-dead" "(std error conditions)") ("&actor-timeout" "(std error conditions)") @@ -16648,11 +16654,18 @@ ("sqlite-bind" "(jerboa prelude safe)" "(jerboa prelude)") ("sqlite-bind!" "(std db sqlite)") ("sqlite-bind-blob" "(std db sqlite-native)") + ("sqlite-bind-blob!" "(std db sqlite)") ("sqlite-bind-double" "(std db sqlite-native)") ("sqlite-bind-int" "(std db sqlite-native)") + ("sqlite-bind-int!" "(std db sqlite)") ("sqlite-bind-null" "(std db sqlite-native)") ("sqlite-bind-null!" "(std db sqlite)") + ("sqlite-bind-parameter-count" "(std db sqlite)") + ("sqlite-bind-parameter-index" "(std db sqlite)") + ("sqlite-bind-parameter-name" "(std db sqlite)") + ("sqlite-bind-real!" "(std db sqlite)") ("sqlite-bind-text" "(std db sqlite-native)") + ("sqlite-bind-text!" "(std db sqlite)") ("sqlite-changes" "(std db sqlite)" "(std db sqlite-native)") @@ -16677,7 +16690,10 @@ "(std db sqlite-native)") ("sqlite-column-value" "(std db sqlite)") ("sqlite-columns" "(std db sqlite)") + ("sqlite-db->bytevector" "(std db sqlite)") + ("sqlite-db?" "(std db sqlite)") ("sqlite-done?" "(std db sqlite-native)") + ("sqlite-errcode" "(std db sqlite)") ("sqlite-errmsg" "(std db sqlite)" "(std db sqlite-native)") ("sqlite-eval" "(std db sqlite)") ("sqlite-exec" @@ -16688,7 +16704,9 @@ ("sqlite-execute" "(jerboa prelude safe)" "(jerboa prelude)" + "(std db sqlite)" "(std db sqlite-native)") + ("sqlite-extended-errcode" "(std db sqlite)") ("sqlite-finalize" "(jerboa prelude safe)" "(jerboa prelude)" @@ -16702,6 +16720,8 @@ "(jerboa prelude)" "(std db sqlite)" "(std db sqlite-native)") + ("sqlite-open-bytevector" "(std db sqlite)") + ("sqlite-open-v2" "(std db sqlite)") ("sqlite-prepare" "(jerboa prelude safe)" "(jerboa prelude)" @@ -16713,12 +16733,15 @@ "(std db sqlite)" "(std db sqlite-native)") ("sqlite-reset" "(std db sqlite)" "(std db sqlite-native)") + ("sqlite-row" "(std db sqlite)") ("sqlite-row?" "(std db sqlite-native)") ("sqlite-step" "(jerboa prelude safe)" "(jerboa prelude)" "(std db sqlite)" "(std db sqlite-native)") + ("sqlite-stmt?" "(std db sqlite)") + ("sqlite-total-changes" "(std db sqlite)") ("srange" "(std specter)") ("sre->named-groups" "(std srfi srfi-115)") ("sre->pattern-string" "(std srfi srfi-115)") --- a/data/changelog.sexp +++ b/data/changelog.sexp @@ -2,7 +2,26 @@ . "Machine-readable changelog of Jerboa API drift. Consumers (LLM tooling, lints, jerboa_verify) use this to invalidate stale recommendations and to suggest migrations when a symbol is renamed or relocated.") ("entries" - (("added" "*csv-max-field-length*" "*json-max-total-bytes*") + (("added" "sqlite-open-v2" "sqlite-open-bytevector" + "sqlite-db->bytevector" "sqlite-db?" "sqlite-execute" + "sqlite-stmt?" "sqlite-bind-int!" "sqlite-bind-real!" + "sqlite-bind-text!" "sqlite-bind-blob!" + "sqlite-bind-parameter-count" "sqlite-bind-parameter-name" + "sqlite-bind-parameter-index" "sqlite-row" + "sqlite-total-changes" "sqlite-errcode" + "sqlite-extended-errcode") + ("date" . "2026-06-14") + ("modules_added") + ("moved") + ("notes" + . + "(std db sqlite) now wraps the vendored pure-Jerboa jsqlite engine instead of the old external SQLite shim. The default Makefile libdirs include vendor/jsqlite/src, prelude safe SQLite aliases lazily resolve through (std db sqlite), and data/api-signatures.sexp plus LSP completion tables were refreshed from source: 707 modules, 10,267 symbols, 13,466 exports, and 0 parse errors.") + ("removed") + ("renamed") + ("tier_changes") + ("tools_added") + ("version" . #f)) + (("added" "*csv-max-field-length*" "*json-max-total-bytes*") ("date" . "2026-06-11") ("modules_added") ("moved") --- a/data/cookbooks.sexp +++ b/data/cookbooks.sexp @@ -1160,7 +1160,7 @@ "Sub-bytevector helper avoiding R6RS bytevector-copy warning")) (("code" . - "(import (std db sqlite-native))\n\n;; Open in-memory database\n(define db (sqlite-open \":memory:\"))\n\n;; Create table\n(sqlite-exec db \"CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, age INTEGER)\")\n\n;; Insert with parameters (type auto-detected)\n(sqlite-execute db \"INSERT INTO users VALUES (?, ?, ?)\" 1 \"Alice\" 30)\n(sqlite-execute db \"INSERT INTO users VALUES (?, ?, ?)\" 2 \"Bob\" 25)\n\n;; Query returns list of alists\n(define rows (sqlite-query db \"SELECT name, age FROM users WHERE age > ?\" 20))\n;; => ((\"name\" . \"Alice\") (\"age\" . 30)) ((\"name\" . \"Bob\") (\"age\" . 25)))\n\n(for-each\n (lambda (row)\n (format #t \"~a: ~a\\n\" (cdr (assoc \"name\" row)) (cdr (assoc \"age\" row))))\n rows)\n\n;; Low-level prepared statement API\n(define stmt (sqlite-prepare db \"SELECT * FROM users WHERE id = ?\"))\n(sqlite-bind-int stmt 1 1)\n(let ([rc (sqlite-step stmt)])\n (when (sqlite-row? rc)\n (format #t \"Found: ~a\\n\" (sqlite-column-text stmt 1))))\n(sqlite-finalize stmt)\n\n(sqlite-close db)") ("id" . "jerboa-sqlite-native-basic") + "(import (std db sqlite-native))\n\n;; Open in-memory database\n(define db (sqlite-open \":memory:\"))\n\n;; Create table\n(sqlite-exec db \"CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, age INTEGER)\")\n\n;; Insert with parameters (type auto-detected)\n(sqlite-execute db \"INSERT INTO users VALUES (?, ?, ?)\" 1 \"Alice\" 30)\n(sqlite-execute db \"INSERT INTO users VALUES (?, ?, ?)\" 2 \"Bob\" 25)\n\n;; Query returns list of alists\n(define rows (sqlite-query db \"SELECT name, age FROM users WHERE age > ?\" 20))\n;; => ((\"name\" . \"Alice\") (\"age\" . 30)) ((\"name\" . \"Bob\") (\"age\" . 25)))\n\n(for-each\n (lambda (row)\n (format #t \"~a: ~a\\n\" (cdr (assoc \"name\" row)) (cdr (assoc \"age\" row))))\n rows)\n\n;; Low-level prepared statement API\n(define stmt (sqlite-prepare db \"SELECT * FROM users WHERE id = ?\"))\n(sqlite-bind-int stmt 1 1)\n(let ([rc (sqlite-step stmt)])\n (when (sqlite-row? rc)\n (format #t \"Found: ~a\\n\" (sqlite-column-text stmt 1))))\n(sqlite-finalize stmt)\n\n(sqlite-close db)") ("id" . "sqlite-native-basic") ("imports" "(std db sqlite-native)") ("notes" . @@ -3323,7 +3323,7 @@ ("imports" "(std db sqlite-native)" "(std stm)") ("notes" . - "Use (std db sqlite-native) NOT (std db sqlite) — the sqlite module is missing (jerboa-sqlite library not bundled). sqlite-native uses bundled rusqlite with no external libsqlite3 dependency. sqlite-query returns a list of alists where keys are column name strings — use (assoc \"column-name\" row) with string keys, not symbol keys. sqlite-execute auto-detects parameter types: integers, flonums, strings, bytevectors are bound natively; #f binds as NULL. WAL mode is critical for concurrent service workloads — without it, any reader blocks all writers. Fire-and-forget guard pattern ((guard (e [#t (void)]))) ensures a SQLite error never crashes a worker thread processing an event. Flush WAL at shutdown: PRAGMA wal_checkpoint(FULL) blocks until all WAL pages are written back to the main DB file. sqlite-open \":memory:\" for in-memory databases.") + "Use (std db sqlite) for the default vendored jsqlite backend. Use (std db sqlite-native) only when you explicitly need the Rust native backend. sqlite-native uses bundled rusqlite with no external libsqlite3 dependency. sqlite-query returns a list of alists where keys are column name strings — use (assoc \"column-name\" row) with string keys, not symbol keys. sqlite-execute auto-detects parameter types: integers, flonums, strings, bytevectors are bound natively; #f binds as NULL. WAL mode is critical for concurrent service workloads — without it, any reader blocks all writers. Fire-and-forget guard pattern ((guard (e [#t (void)]))) ensures a SQLite error never crashes a worker thread processing an event. Flush WAL at shutdown: PRAGMA wal_checkpoint(FULL) blocks until all WAL pages are written back to the main DB file. sqlite-open \":memory:\" for in-memory databases.") ("tags" "sqlite" "sqlite-native" "wal" "persistence" "stm" "replay" "startup" "std db sqlite-native") ("title" @@ -4176,7 +4176,7 @@ "Resolve jerbuild auto-import conflicts between (jerboa core) and (jerboa runtime)")) (("code" . - "# Before building a binary for a different Chez machine type, remove compiled caches\n# from every library root that can be embedded in the boot image.\nfor d in src/jsh src/compat vendor/jerboa/lib vendor/jerboa-ssh/src \\\n vendor/jerboa-sqlite/src vendor/jerboa-crypto/src \\\n vendor/jerboa-coreutils/lib vendor/jerboa-awk/lib \\\n vendor/jerboa-sed/lib vendor/jerboa-aws/lib vendor/jerboa-fuse/lib; do\n [ -d \"$d\" ] && find \"$d\" \\( -name '*.so' -o -name '*.wpo' \\) -delete\ndone\nrm -f jsh-generated.so jsh-generated.wpo jsh.boot jsh-libs.boot") ("id" . "clean-cross-target-fasl-caches") ("imports") + "# Before building a binary for a different Chez machine type, remove compiled caches\n# from every library root that can be embedded in the boot image.\nfor d in src/jsh src/compat vendor/jerboa/lib vendor/jerboa-ssh/src \\\n vendor/jsqlite/src vendor/jerboa-crypto/src \\\n vendor/jerboa-coreutils/lib vendor/jerboa-awk/lib \\\n vendor/jerboa-sed/lib vendor/jerboa-aws/lib vendor/jerboa-fuse/lib; do\n [ -d \"$d\" ] && find \"$d\" \\( -name '*.so' -o -name '*.wpo' \\) -delete\ndone\nrm -f jsh-generated.so jsh-generated.wpo jsh.boot jsh-libs.boot") ("id" . "clean-cross-target-fasl-caches") ("imports") ("notes" . "A Linux cross-build can leave ta6le/tarm64le FASLs in vendored library trees. A later macOS build may embed those stale files and fail at runtime with \"incompatible fasl-object machine-type\" or compilation-instance errors. Clean all library roots that make-boot-file can consume, not only project-local src/.") @@ -4750,10 +4750,10 @@ "Compile a Jerboa project without running its CLI entrypoint")) (("code" . - ";; support/sqlite-bundled/Cargo.toml\n[package]\nname = \"app-sqlite-bundled\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\ncrate-type = [\"staticlib\"]\n\n[dependencies]\nlibsqlite3-sys = { version = \"0.30\", features = [\"bundled\"] }\n\n;; support/sqlite-bundled/src/lib.rs\n#[no_mangle]\npub extern \"C\" fn app_sqlite_bundled_anchor() -> i32 {\n unsafe { libsqlite3_sys::sqlite3_libversion_number() }\n}\n\n;; .jerbuild\n(entry \"main.ss\")\n(output \"app\")\n(ffi-symbols \"support/ffi-symbols.list\")\n(extra-sources\n (\"vendor/jerboa-sqlite/jerboa_sqlite_shim.c\" cflags: \"-Isupport\"))\n(extra-archives\n \"support/sqlite-bundled/target/release/libapp_sqlite_bundled.a\")\n\n;; Makefile\nsupport/sqlite-bundled/target/release/libapp_sqlite_bundled.a:\n\tcargo build --manifest-path support/sqlite-bundled/Cargo.toml --release\n\nbinary: support/sqlite-bundled/target/release/libapp_sqlite_bundled.a\n\tjerbuild build --config .jerbuild --os-libs \"-lm -ldl -lpthread -luuid -lncurses -lstdc++\"\n") ("id" . "jerbuild-bundled-sqlite-shim") ("imports") + ";; support/sqlite-bundled/Cargo.toml\n[package]\nname = \"app-sqlite-bundled\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\ncrate-type = [\"staticlib\"]\n\n[dependencies]\nlibsqlite3-sys = { version = \"0.30\", features = [\"bundled\"] }\n\n;; support/sqlite-bundled/src/lib.rs\n#[no_mangle]\npub extern \"C\" fn app_sqlite_bundled_anchor() -> i32 {\n unsafe { libsqlite3_sys::sqlite3_libversion_number() }\n}\n\n;; .jerbuild\n(entry \"main.ss\")\n(output \"app\")\n(ffi-symbols \"support/ffi-symbols.list\")\n(extra-sources\n (\"vendor/jsqlite/shim/driver.c\" cflags: \"-Isupport\"))\n(extra-archives\n \"support/sqlite-bundled/target/release/libapp_sqlite_bundled.a\")\n\n;; Makefile\nsupport/sqlite-bundled/target/release/libapp_sqlite_bundled.a:\n\tcargo build --manifest-path support/sqlite-bundled/Cargo.toml --release\n\nbinary: support/sqlite-bundled/target/release/libapp_sqlite_bundled.a\n\tjerbuild build --config .jerbuild --os-libs \"-lm -ldl -lpthread -luuid -lncurses -lstdc++\"\n") ("id" . "jerbuild-vendored-jsqlite") ("imports") ("notes" . - "This keeps user repos buildable with jerbuild plus a C compiler and Rust/Cargo, without requiring system sqlite headers or libsqlite3 packages. Put a minimal sqlite3.h in support/ if the shim only needs a small API surface, add -Isupport on the shim source, and list the bundled Rust static archive in extra-archives so sqlite symbols are available at final link.") + "Use the vendored jsqlite sources through the Jerboa library path instead of carrying a libsqlite3 shim. Add vendor/jsqlite/src to libdirs for execution and binary builds.") ("tags" "jerbuild" "sqlite" "ffi" "staticlib" "extra-archives" "c-shim") ("title" @@ -4904,7 +4904,7 @@ "Persist records portably by decomposing to plain data (avoid fasl pitfalls)")) (("code" . - "jerbuild exec --libdirs ./lib:vendor/jerboa-sqlite/lib:vendor/jerboa-websearch/src:$(jerbuild --jerboa-home)/lib /dev/stdin <<'EOF'\n(import (jerboa prelude) (jcode ui tui-theme))\n(displayln (get-registered-themes))\n(displayln (set-theme-by-name! \"opencode-dark\"))\n(displayln (current-theme-name))\nEOF") ("id" . "jerbuild-exec-local-module-smoke-test") + "jerbuild exec --libdirs ./lib:vendor/jsqlite/src:vendor/jerboa-websearch/src:$(jerbuild --jerboa-home)/lib /dev/stdin <<'EOF'\n(import (jerboa prelude) (jcode ui tui-theme))\n(displayln (get-registered-themes))\n(displayln (set-theme-by-name! \"opencode-dark\"))\n(displayln (current-theme-name))\nEOF") ("id" . "jerbuild-exec-local-module-smoke-test") ("imports") ("notes" . --- a/docs/architecture-split.md +++ b/docs/architecture-split.md @@ -75,7 +75,7 @@ These entries should be split over time into two groups: | jerboa-crypto (OpenSSL) | ring | `(std crypto native-rust)` | | chez-zlib (libz) | flate2 | `(std compress native-rust)` | | jerboa-pcre2 (libpcre2) | regex (NFA, ReDoS-immune) | `(std regex-native)` | -| jerboa-sqlite (libsqlite3) | rusqlite (bundled) | `(std db sqlite-native)` | +| legacy libsqlite3 SQLite wrapper (removed) | jsqlite (vendored pure Jerboa) | `(std db sqlite)` | | chez-postgresql (libpq) | rust-postgres | `(std db postgresql-native)` | | chez-epoll (syscalls) | libc (Rust) | `(std os epoll-native)` | | jerboa-inotify (syscalls) | libc (Rust) | `(std os inotify-native)` | --- a/docs/jerboa-code-typed-static-plan.md +++ b/docs/jerboa-code-typed-static-plan.md @@ -130,7 +130,8 @@ Avoid these as the first pilot: network boundaries. - `src/jcode/ui/tui-ffi.ss`: terminal FFI and C shim work should remain a separate audited boundary. -- `vendor/jerboa-sqlite`: SQLite is a deliberate native database boundary. +- `vendor/jsqlite`: SQLite is a large database subsystem; tackle it separately + from the first typed-code pilot. - `vendor/jerboa-websearch/wasm`: handwritten Rust/WASM, but better tackled after the project attachment path exists. --- a/docs/libraries.md +++ b/docs/libraries.md @@ -37,7 +37,7 @@ Gerbil has ~438 `:std/*` modules. Jerboa currently implements 51. This document | [chez-epoll](https://git.sr.ht/~lisp/chez-epoll) | Linux epoll | `(std os epoll)` | Done | | [jerboa-inotify](https://git.sr.ht/~lisp/jerboa-inotify) | Linux inotify | `(std os inotify)` | Done | | [jerboa-crypto](https://git.sr.ht/~lisp/jerboa-crypto) | OpenSSL EVP | `(std crypto cipher)`, `(std crypto hmac)`, `(std crypto pkey)`, `(std crypto kdf)`, `(std crypto etc)` | Done | -| [jerboa-sqlite](https://git.sr.ht/~lisp/jerboa-sqlite) | SQLite3 | `(std db sqlite)` | Done | +| [jsqlite](../vendor/jsqlite) | SQLite3 | `(std db sqlite)` | Vendored | | [chez-postgresql](https://git.sr.ht/~lisp/chez-postgresql) | libpq | `(std db postgresql)` | Done | ### Replaced (No Longer Needed) --- a/docs/native-rust.md +++ b/docs/native-rust.md @@ -69,7 +69,7 @@ Replacing them with a single Rust shared library (`libjerboa_native.so`) provide |----------------------------|-----------------|---------------------|-----------------------------| | **libz.so** | chez-zlib | `std/compress/zlib` | Deflate/inflate compression | | **libpcre2-8.so** | jerboa-pcre2 | `std/pcre2` | Perl-compatible regex | -| **libsqlite3.so** | jerboa-sqlite | `std/db/sqlite` | SQLite database | +| ~~**libsqlite3.so**~~ | ~~SQLite C shim~~ | `std/db/sqlite` | **Eliminated** — default SQLite now uses vendored pure Jerboa `jsqlite` | | **libpq.so** | chez-postgresql | `std/db/postgresql` | PostgreSQL client | | **libleveldb.so** | chez-leveldb | `std/db/leveldb` | LevelDB key-value store | | ~~**libyaml.so**~~ | ~~jerboa-yaml~~ | `std/text/yaml` | ~~YAML parsing~~ — **Eliminated**: now pure Scheme with roundtrip support | @@ -166,11 +166,11 @@ The pure Scheme implementation provides: - Node manipulation: `yaml-mapping-ref`, `yaml-mapping-set!`, `yaml-ref`, `yaml-set!` - Security: `*yaml-max-input-size*` (1MB) and `*yaml-max-depth*` (512) limits -### Databases: libsqlite3 → rusqlite, libpq → rust-postgres +### Databases: jsqlite default, libpq → rust-postgres | Current | Replacement | Notes | |---------|-------------|-------| -| libsqlite3 (via jerboa-sqlite) | **rusqlite** | Bundles SQLite source or links system lib. Parameterized queries by default. | +| SQLite default | **jsqlite** | Vendored pure Jerboa engine used by `(std db sqlite)`. `(std db sqlite-native)` remains an explicit Rust/rusqlite backend. | | libpq (via chez-postgresql) | **rust-postgres** | Pure Rust PostgreSQL client. TLS via rustls. No C dependency. | | libleveldb (via chez-leveldb) | **rusty-leveldb** or **sled** | rusty-leveldb is API-compatible; sled is pure Rust with different (better) API | --- a/lib/std/db/sqlite.ss +++ b/lib/std/db/sqlite.ss @@ -1,26 +1,36 @@ #!chezscheme -;;; :std/db/sqlite -- SQLite3 database (wraps jerboa-sqlite) -;;; Requires: jerboa_sqlite_shim.so, libsqlite3.so +;;; :std/db/sqlite -- SQLite3 database (vendored jsqlite) (library (std db sqlite) (export ;; Core - sqlite-open sqlite-close - sqlite-exec sqlite-eval sqlite-query + sqlite-open sqlite-open-v2 sqlite-open-bytevector sqlite-db->bytevector + sqlite-close sqlite-db? + sqlite-exec sqlite-eval sqlite-execute sqlite-query ;; Prepared statements sqlite-prepare sqlite-finalize - sqlite-step sqlite-reset sqlite-clear-bindings + sqlite-step sqlite-reset sqlite-stmt? sqlite-clear-bindings ;; Binding sqlite-bind! sqlite-bind-null! + sqlite-bind-int! sqlite-bind-real! sqlite-bind-text! sqlite-bind-blob! + sqlite-bind-parameter-count sqlite-bind-parameter-name sqlite-bind-parameter-index ;; Column access sqlite-column-count sqlite-column-name sqlite-column-type - sqlite-column-value sqlite-columns + sqlite-column-value sqlite-columns sqlite-row ;; Metadata - sqlite-last-insert-rowid sqlite-changes sqlite-errmsg + sqlite-last-insert-rowid sqlite-changes sqlite-total-changes + sqlite-errcode sqlite-extended-errcode sqlite-errmsg ;; Constants SQLITE_ROW SQLITE_DONE SQLITE_OK SQLITE_INTEGER SQLITE_FLOAT SQLITE_TEXT SQLITE_BLOB SQLITE_NULL) - (import (jerboa-sqlite)) + (import (only (jerboa core) def) + (jsqlite api) + (only (jsqlite constants) + SQLITE_ROW SQLITE_DONE SQLITE_OK + SQLITE_INTEGER SQLITE_FLOAT SQLITE_TEXT SQLITE_BLOB SQLITE_NULL)) + + (def sqlite-eval sqlite-exec) + (def sqlite-execute sqlite-exec) ) ;; end library --- a/lib/std/safe.ss +++ b/lib/std/safe.ss @@ -165,8 +165,9 @@ ;; SQLite — Contract-checked wrappers ;; ========================================================================= ;; - ;; These are *forward declarations* — they attempt to load the native module - ;; and wrap it. If sqlite-native is not available, they raise a clear error. + ;; These are *forward declarations* — they lazily load the vendored jsqlite + ;; module and wrap it. Lazy resolution avoids an import cycle while the + ;; prelude is loading. (def sqlite-available? #f) (def raw-sqlite-open #f) @@ -177,16 +178,16 @@ (def raw-sqlite-prepare #f) (def raw-sqlite-finalize #f) (def raw-sqlite-step #f) - (def raw-sqlite-bind-int #f) - (def raw-sqlite-bind-double #f) - (def raw-sqlite-bind-text #f) - (def raw-sqlite-bind-null #f) + (def raw-sqlite-bind! #f) + (def raw-sqlite-bind-null! #f) (def raw-sqlite-errmsg #f) + (def raw-sqlite-db? #f) + (def raw-sqlite-stmt? #f) - ;; Try to load sqlite bindings at library init time. ;; Set sqlite-available? LAST so partial failure leaves it #f. - (def _init-sqlite - (try (let ([env (environment '(std db sqlite-native))]) + (def (init-sqlite!) + (unless sqlite-available? + (try (let ([env (environment '(std db sqlite))]) (set! raw-sqlite-open (eval 'sqlite-open env)) (set! raw-sqlite-close (eval 'sqlite-close env)) (set! raw-sqlite-exec (eval 'sqlite-exec env)) @@ -195,21 +196,32 @@ (set! raw-sqlite-prepare (eval 'sqlite-prepare env)) (set! raw-sqlite-finalize (eval 'sqlite-finalize env)) (set! raw-sqlite-step (eval 'sqlite-step env)) - (set! raw-sqlite-bind-int (eval 'sqlite-bind-int env)) - (set! raw-sqlite-bind-double (eval 'sqlite-bind-double env)) - (set! raw-sqlite-bind-text (eval 'sqlite-bind-text env)) - (set! raw-sqlite-bind-null (eval 'sqlite-bind-null env)) + (set! raw-sqlite-bind! (eval 'sqlite-bind! env)) + (set! raw-sqlite-bind-null! (eval 'sqlite-bind-null! env)) (set! raw-sqlite-errmsg (eval 'sqlite-errmsg env)) + (set! raw-sqlite-db? (eval 'sqlite-db? env)) + (set! raw-sqlite-stmt? (eval 'sqlite-stmt? env)) ;; Only mark available after ALL evals succeed (set! sqlite-available? #t)) - (catch (exn) (set! sqlite-available? #f) (void)))) + (catch (exn) (set! sqlite-available? #f) (void))))) (def (ensure-sqlite! who) + (init-sqlite!) (unless sqlite-available? (raise (condition (make-db-error 'db 'sqlite) (make-message-condition - (format #f "~a: SQLite not available — libjerboa_native.so not loaded" who)))))) + (format #f "~a: SQLite not available — vendored jsqlite could not be loaded" who)))))) + + (def (check-sqlite-db! who val) + (when-checking + (unless (and raw-sqlite-db? (raw-sqlite-db? val)) + (error who "expected sqlite database, got ~a" val)))) + + (def (check-sqlite-stmt! who val) + (when-checking + (unless (and raw-sqlite-stmt? (raw-sqlite-stmt? val)) + (error who "expected sqlite statement, got ~a" val)))) ;; ---- SQL injection heuristic detection ---- ;; Reject SQL strings that look like they were built by concatenation. @@ -268,7 +280,7 @@ (ensure-sqlite! 'safe-sqlite-open) (let ([handle (raw-sqlite-open path)]) (when-checking - (when (and (fixnum? handle) (fx< handle 0)) + (unless (and raw-sqlite-db? (raw-sqlite-db? handle)) (raise (condition (make-db-connection-error 'db 'sqlite) (make-message-condition @@ -282,9 +294,8 @@ handle)) (def (safe-sqlite-close db) - ;; Pre: db must be a fixnum handle - (check-fixnum! 'safe-sqlite-close db) (ensure-sqlite! 'safe-sqlite-close) + (check-sqlite-db! 'safe-sqlite-close db) ;; Mark as closed so guardian won't warn (let ([entry (hashtable-ref *sqlite-handle-entries* db #f)]) (mark-resource-closed! entry) @@ -292,13 +303,13 @@ (raw-sqlite-close db)) (def (safe-sqlite-exec db sql) - ;; Pre: db is fixnum handle, sql is string, not tainted + ;; Pre: db is sqlite database, sql is string, not tainted ;; Post: returns 0 on success (check-untainted! sql 'sqlite-exec) - (check-fixnum! 'safe-sqlite-exec db) + (ensure-sqlite! 'safe-sqlite-exec) + (check-sqlite-db! 'safe-sqlite-exec db) (check-string! 'safe-sqlite-exec sql) (check-sql-safety! 'safe-sqlite-exec sql) - (ensure-sqlite! 'safe-sqlite-exec) (let ([rc (raw-sqlite-exec db sql)]) (when-checking (unless (and (fixnum? rc) (fx= rc 0)) @@ -312,22 +323,22 @@ rc)) (def (safe-sqlite-execute db sql . params) - ;; Pre: db is fixnum handle, sql is string, params is list - (check-fixnum! 'safe-sqlite-execute db) + ;; Pre: db is sqlite database, sql is string, params is list + (ensure-sqlite! 'safe-sqlite-execute) + (check-sqlite-db! 'safe-sqlite-execute db) (check-string! 'safe-sqlite-execute sql) (check-untainted! 'safe-sqlite-execute sql) (check-sql-safety! 'safe-sqlite-execute sql) - (ensure-sqlite! 'safe-sqlite-execute) (apply raw-sqlite-execute db sql params)) (def (safe-sqlite-query db sql . params) - ;; Pre: db is fixnum handle, sql is string - ;; Post: returns a list of alists - (check-fixnum! 'safe-sqlite-query db) + ;; Pre: db is sqlite database, sql is string + ;; Post: returns a list of rows + (ensure-sqlite! 'safe-sqlite-query) + (check-sqlite-db! 'safe-sqlite-query db) (check-string! 'safe-sqlite-query sql) (check-untainted! 'safe-sqlite-query sql) (check-sql-safety! 'safe-sqlite-query sql) - (ensure-sqlite! 'safe-sqlite-query) (let ([result (apply raw-sqlite-query db sql params)]) (when-checking (unless (list? result) @@ -337,14 +348,14 @@ result)) (def (safe-sqlite-prepare db sql) - (check-fixnum! 'safe-sqlite-prepare db) + (ensure-sqlite! 'safe-sqlite-prepare) + (check-sqlite-db! 'safe-sqlite-prepare db) (check-string! 'safe-sqlite-prepare sql) (check-untainted! 'safe-sqlite-prepare sql) (check-sql-safety! 'safe-sqlite-prepare sql) - (ensure-sqlite! 'safe-sqlite-prepare) (let ([stmt (raw-sqlite-prepare db sql)]) (when-checking - (when (and (fixnum? stmt) (fx< stmt 0)) + (unless (and raw-sqlite-stmt? (raw-sqlite-stmt? stmt)) (raise (condition (make-db-query-error 'db 'sqlite sql) (make-message-condition @@ -355,31 +366,27 @@ stmt)) (def (safe-sqlite-finalize stmt) - (check-fixnum! 'safe-sqlite-finalize stmt) (ensure-sqlite! 'safe-sqlite-finalize) + (check-sqlite-stmt! 'safe-sqlite-finalize stmt) (raw-sqlite-finalize stmt)) (def (safe-sqlite-step stmt) - (check-fixnum! 'safe-sqlite-step stmt) (ensure-sqlite! 'safe-sqlite-step) + (check-sqlite-stmt! 'safe-sqlite-step stmt) (raw-sqlite-step stmt)) (def (safe-sqlite-bind stmt index value) ;; Dispatches to the right bind function based on value type. - (check-fixnum! 'safe-sqlite-bind stmt) - (check-fixnum! 'safe-sqlite-bind index) (ensure-sqlite! 'safe-sqlite-bind) + (check-sqlite-stmt! 'safe-sqlite-bind stmt) + (check-fixnum! 'safe-sqlite-bind index) (cond - [(fixnum? value) (raw-sqlite-bind-int stmt index value)] - [(flonum? value) (raw-sqlite-bind-double stmt index value)] - [(string? value) (raw-sqlite-bind-text stmt index value)] - [(not value) (raw-sqlite-bind-null stmt index)] - [(bytevector? value) - ;; No blob bind available in current API — convert to string - (error 'safe-sqlite-bind "bytevector binding not yet supported")] + [(not value) (raw-sqlite-bind-null! stmt index)] + [(or (integer? value) (flonum? value) (string? value) (bytevector? value)) + (raw-sqlite-bind! stmt index value)] [else (error 'safe-sqlite-bind - "unsupported bind type: ~a (expected fixnum, flonum, string, or #f)" + "unsupported bind type: ~a (expected integer, flonum, string, bytevector, or #f)" value)])) ;; ========================================================================= --- a/tests/test-wrapper-sqlite.ss +++ b/tests/test-wrapper-sqlite.ss @@ -14,12 +14,12 @@ (display " => ") (write r) (display " expected ") (write e) (newline))))])) -(define test-db (make-temporary-file-name "jerboa-sqlite-test")) +(define test-db (make-temporary-file-name "jsqlite-test")) (when (file-exists? test-db) (delete-file test-db)) ;; Open (define db (sqlite-open test-db)) -(chk (not (zero? db)) => #t) +(chk (sqlite-db? db) => #t) ;; Create + insert (sqlite-exec db "CREATE TABLE t (id INTEGER PRIMARY KEY, val TEXT)") @@ -31,8 +31,8 @@ ;; Query (let ([rows (sqlite-query db "SELECT id, val FROM t ORDER BY id")]) (chk (length rows) => 2) - (chk (vector-ref (car rows) 1) => "hello") - (chk (vector-ref (cadr rows) 1) => "world")) + (chk (list-ref (car rows) 1) => "hello") + (chk (list-ref (cadr rows) 1) => "world")) ;; Cleanup (sqlite-close db) new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/.gitignore @@ -0,0 +1,20 @@ +# Transpiled / compiled artifacts +/lib/ +*.sls +*.so +*.dylib +*.wpo +**/.jerbuild-hashes + +# OS / editor +.DS_Store +*~ + +# Local scratch logs (never committed) +/.scratch/ + +# Database sidecar files created at runtime (locking / WAL / rollback journal) +*-lock +*-wal +*-shm +*-journal new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/Makefile @@ -0,0 +1,89 @@ +# jsqlite -- a SQLite3-compatible database engine in Jerboa Scheme. +# +# jerbuild bundles Chez Scheme + the jerboa stdlib, so building jsqlite needs +# only `jerbuild` on PATH. This vendored copy keeps only self-contained +# production, unit, and robustness targets. + +JERBUILD ?= jerbuild +JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) +ifeq ($(JH),) +$(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) +endif + +SRC_DIR := $(CURDIR)/src +TOOLS_DIR := $(CURDIR)/tools + +# Production execution: just our source + the jerboa stdlib. +LIBDIRS := $(SRC_DIR):$(JH)/lib + +UNIT_TESTS := $(wildcard tests/unit/*.ss) + +.PHONY: all build unit test corrupt concurrency cshim robustness clean help +.DEFAULT_GOAL := help + +all: test + +# Compile-check: import every production module (R6RS libraries in .ss files +# are consumed directly via libdirs, so there is no transpile step). +build: + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/build-check.ss + +# Unit tests run directly against src/ (jerbuild exec compiles on the fly). +unit: + @fail=0; for t in $(UNIT_TESTS); do \ + echo "== unit: $$t =="; \ + $(JERBUILD) exec --libdirs "$(LIBDIRS)" $$t || fail=1; \ + done; \ + if [ $$fail -ne 0 ]; then echo "UNIT FAILURES"; exit 1; fi + +test: unit + +# Malformed-image corpus test. +corrupt: + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/corrupt-test.ss $(N) $(SEED) + +# Multi-process write-locking stress test: 4 processes hammer one file, then +# verify no writes were lost. Needs only jsqlite (no oracle). `make concurrency N=40`. +concurrency: + @P=/tmp/jsqlite-concurrency.db; n=$${N:-40}; \ + for mode in "" wal; do \ + echo "== concurrency (journal mode: $${mode:-rollback}) =="; \ + rm -f $$P $$P-lock $$P-journal $$P-wal $$P-shm; \ + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss init $$P; \ + for id in 1 2 3 4; do \ + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss worker $$id $$n $$P $$mode >/dev/null & \ + done; wait; \ + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss verify $$P 4 $$n || exit 1; \ + done + +# Write-path / B-tree property test vs the oracle (`make dmlfuzz N=2000 SEED=7`). +dmlfuzz: + @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP dmlfuzz: reference $(REF_DIR) not found"; exit 0; fi; \ + $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ + { echo "SKIP dmlfuzz: reference build failed"; exit 0; }; \ + DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ + $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/dml-fuzz.ss $(N) $(SEED) + +# C ABI shim: compile the C driver (shim/driver.c) and have it drive jsqlite +# prepared statements through the jsq_api function-pointer table. Needs a C +# compiler (cc); no oracle. +cshim: + $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/cshim-test.ss + +robustness: corrupt concurrency + +clean: + rm -rf lib + find . \( -name '*.wpo' -o -name '*.so' \) -delete 2>/dev/null || true + +help: + @echo "jsqlite -- SQLite3-compatible engine in Jerboa Scheme" + @echo "" + @echo " make build Compile-check all production modules" + @echo " make unit Run unit tests (tests/unit/*.ss)" + @echo " make test Run unit tests (default acceptance gate)" + @echo " make corrupt Malformed-database corpus robustness test (N=, SEED=)" + @echo " make concurrency Multi-process write-locking stress test (N=)" + @echo " make cshim C ABI shim: a C driver runs jsqlite prepared statements (needs cc)" + @echo " make robustness corrupt + concurrency" + @echo " make clean Remove build artifacts" new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/README.md @@ -0,0 +1,114 @@ +# jsqlite + +A SQLite3-compatible database engine implemented from scratch in **Jerboa +Scheme** — not a binding over `libsqlite3`, but a reimplementation of the +engine: tokenizer, parser, value/type system, VDBE-style virtual machine, +B-tree storage, pager, and the `sqlite3_*`-shaped API. + +Pinned compatibility target: **SQLite 3.54.0**. This vendored copy is used as a +runtime dependency directly; it does not require libsqlite3. + +## Status + +Built in phases in the upstream jsqlite project. + +| Phase | Scope | State | +|------|-------|-------| +| 0 | Scaffold, value/type foundations, oracle harness | ✅ done | +| 1 | Values, tokenizer, parser, in-memory evaluator | ✅ done | +| 2 | SQLite file reader (header, records, B-tree, schema) | ✅ done | +| 3 | Read-only SELECT over tables (WHERE/ORDER/LIMIT/DISTINCT/joins) | ✅ done | +| 4 | Write path: CREATE TABLE/CTAS / CREATE/DROP INDEX / INSERT / UPDATE / DELETE, NOT NULL, freelist basics (experimental) | ✅ done | +| 7a | Aggregates + GROUP BY/HAVING; BETWEEN/IN/LIKE/GLOB/COLLATE + scalar functions; compound UNION/INTERSECT/EXCEPT; derived tables, non-recursive CTEs, `WITH` on INSERT SELECT, and views | ✅ done | +| next | Remaining foreign-key edge cases, index planner, locking/fsync/WAL hardening | planned | + +What works today: the full value/type system (NULL/INTEGER/REAL/TEXT/BLOB with +SQLite truthiness, numeric coercion, storage-class comparison ordering and +three-valued logic); a complete SQLite tokenizer (SQLite keywords, all literal and +operator forms); a precedence-correct expression parser; and an evaluator +covering arithmetic (with integer-overflow→real and `/0`→NULL), bitwise ops, +`CAST` affinity, SQLite comparison affinity for direct table columns and rowid, +`CASE`, `COLLATE BINARY/NOCASE/RTRIM` including declared column defaults for +direct table scans, bind parameters, and a core set of scalar functions. +The prepared-statement API covers prepare/step/reset/finalize, generic and typed +bind helpers, column accessors, `sqlite-exec`/`sqlite-query`, change counters, +read-only/open-v2 modes, and connection error accessors. It runs table-less `SELECT`s in memory, verified +expression-by-expression against SQLite 3.54.0 in the upstream jsqlite project. +It also **reads real SQLite database files** — parsing the header, +records, and table B-trees (including interior pages and overflow chains) — so +`SELECT <cols/*> FROM <table>` returns rows from files written by upstream +SQLite, byte-for-byte verified against the reference. Top-level, compound, +derived, scalar, and CTE `VALUES` queries are supported. The write path creates +rowid tables (including `AUTOINCREMENT`/`sqlite_sequence`), materializes +`CREATE TABLE ... AS SELECT`/`VALUES`, supports `STRICT` rowid tables and +STORED/VIRTUAL generated columns, stores/query schema-only views, supports +`ALTER TABLE ... RENAME TO ...`, `ALTER TABLE ... ADD COLUMN`, and +`ALTER TABLE ... RENAME COLUMN ... TO ...`, plus added-column CHECK validation +and added-column REFERENCES/default validation, safe `DROP COLUMN` cases, and creates user index +roots (`CREATE INDEX`, `CREATE UNIQUE INDEX`, `DROP INDEX`) while maintaining +those indexes across INSERT/UPDATE/DELETE for files that pass upstream +`PRAGMA integrity_check`, including `INSERT ... SELECT`, `WITH`-fed INSERT SELECT, +`UPDATE ... FROM`, simple UPSERT, DML `RETURNING`, and multi-page user +indexes; low-level table rowid lookups and index B-tree scans handle upstream +overflow payloads, while multi-level and overflow index writes pass upstream +integrity checks. The planner can use simple rowid equality point lookups, +mandatory `INDEXED BY`, simple single-table constrained index scans, and +covered-projection index scans; broader cost-based planning is still future work. +Foreign-key checks are +available when `PRAGMA foreign_keys=ON` for ordinary rowid tables, including +parent-key mismatch validation and immediate `ON DELETE`/`ON UPDATE` +`RESTRICT`/`CASCADE`/`SET NULL`/`SET DEFAULT` actions, including `DROP TABLE`'s +implicit delete behavior; +`DEFERRABLE INITIALLY DEFERRED` constraints are checked at COMMIT or outermost +savepoint RELEASE, and `PRAGMA defer_foreign_keys` can defer otherwise-immediate +checks for the current transaction. File commits use a simple DELETE-mode +rollback journal and recover a leftover hot journal on open; full locking/fsync +and WAL behavior are still future work. + +## Layout + +``` +src/jsqlite/ production engine modules (never touch libsqlite3) + constants.ss result codes, open flags, datatype tags (SQLite 3.54.0) + error.ss error condition type + result-code plumbing + value.ss values, storage classes, truthiness, coercions + keywords.ss SQL keyword table + tokenize.ss hand-coded tokenizer (mirrors src/tokenize.c) + ast.ss parse.ss AST records + precedence-climbing parser + eval.ss expression evaluator (SQLite semantics) + record.ss page.ss varint/record codec; file header + page geometry + btree.ss table/index B-tree read cursors (interior/leaf, overflow, rowid lookup) + schema.ss sqlite_schema reader + CREATE TABLE column resolution + exec.ss SELECT executor (joins, WHERE, ORDER BY, LIMIT, DISTINCT) + writer.ss mutable DB image: header, page alloc, freelist, table/index rebuilds + dml.ss CREATE TABLE/INDEX / INSERT / UPDATE / DELETE execution + api.ss open/prepare/step/column/finalize (sqlite3_*-shaped) +tests/ + unit/ per-module unit tests +docs/ + architecture.md module map + SQLite pipeline + test-plan.md self-contained vendored test lanes + compatibility-matrix.md feature/status tracking +``` + +## Build & test + +Requires `jerbuild` on `PATH`. + +```sh +make unit # unit tests +make test # unit tests (acceptance gate) +make build # compile-check all production modules +``` + +## Design notes + +- **Values** are native Scheme types plus a single `sql-null` sentinel: + INTEGER = exact integer, REAL = flonum, TEXT = string, BLOB = bytevector. + SQLite has no boolean type; TRUE/FALSE are integers 1/0, and NULL drives + three-valued logic. +- **Compatibility is tracked on three surfaces** — SQL behavior, on-disk file + format, and API shape — each with its own tests. See `docs/test-plan.md`. +- The engine mirrors SQLite's architecture (SQL → tokenize → parse → codegen → + VDBE → B-tree → pager) so that hard compatibility cases (affinity, the file + format, transactions) have a place to live correctly. new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/docs/architecture.md @@ -0,0 +1,68 @@ +# Architecture + +jsqlite mirrors SQLite's own architecture rather than interpreting SQL directly, +because full compatibility (affinity, the file format, transactions, triggers) +is far easier across a prepared-statement / virtual-machine boundary. + +## Pipeline + +``` +SQL text + -> tokenizer (tokenize.ss) bytes -> tokens + -> parser (parse.ss) tokens -> AST (ast.ss) + -> code generator (codegen.ss) AST -> VDBE program + -> VDBE (vdbe.ss) register machine, statement lifecycle + |-- expressions (expr.ss), functions (functions.ss) + |-- cursors over... + -> B-tree (btree.ss) table/index cursors + mutations + -> pager (pager.ss) page cache, rollback journal, locks + -> file/VFS (page.ss) header + page encode/decode +``` + +The public API (`api.ss`) is the prepared-statement boundary: `open -> prepare +-> step -> column-* -> reset/finalize`, shaped like `sqlite3_*`. + +## Module map (target) + +| Module | Responsibility | State | +|--------|----------------|-------| +| `constants.ss` | result codes, open flags, datatype tags | ✅ | +| `error.ss` | error condition type, primary/extended codes | ✅ | +| `value.ss` | storage classes, truthiness, numeric coercion | ✅ (core) | +| `api.ss` | database/statement lifecycle, transactions, utility statements | ✅ (bootstrap) | +| `encoding.ss` | UTF-8 (UTF-16 later) | planned | +| `collation.ss` | BINARY / NOCASE / RTRIM | implemented in `eval.ss`/schema metadata; standalone module/custom collations later | +| `tokenize.ss` | SQLite tokenizer | Phase 1 | +| `parse.ss` / `ast.ss` | parser + AST | Phase 1 | +| `expr.ss` | expression analysis / eval | Phase 1 | +| `vdbe.ss` / `codegen.ss` | VM + code generation | Phase 1 | +| `record.ss` | record varints + serial types | Phase 2 | +| `page.ss` / `btree.ss` | file format + table/index B-tree scans and rowid lookup | Phase 2/4 | +| `pager.ss` / `wal.ss` | pager, journal, locks, WAL | Phase 4/6/9 | +| `schema.ss` | sqlite_schema cache, column/constraint metadata | Phase 2/3/5 | +| `planner.ss` | WHERE / index selection | Phase 3/5 | +| `functions.ss` | scalar/aggregate/window functions | Phase 1/7 | +| `pragma.ss` | PRAGMA handlers | Phase 8 (table/index introspection, cookies, journal/foreign-key toggles) | + +## Value representation + +A SQLite value is one of five storage classes, represented with native Scheme +types plus a single NULL sentinel (`value.ss`): + +| Storage class | Jerboa representation | +|---------------|-----------------------| +| NULL | the unique `sql-null` object | +| INTEGER | exact integer (64-bit signed range enforced in arithmetic) | +| REAL | flonum (IEEE-754 binary64) | +| TEXT | string | +| BLOB | bytevector | + +SQLite has no boolean type. `sql-truthy?` returns `#t`/`#f`/`'null` to model +three-valued logic. TEXT/BLOB are coerced to numbers by their longest leading +numeric run (verified ground truth: `'3abc'` is true, `'0abc'`/`'abc'` false; +a blob's bytes are read as text, so `x'31'` = `"1"` is true). + +## Working rule + +This vendored copy does not call `libsqlite3`. Production modules under +`src/jsqlite/` are the runtime engine. new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/docs/compatibility-matrix.md @@ -0,0 +1,135 @@ +# Compatibility matrix + +Pinned to **SQLite 3.54.0**. Tracks feature status across the three compliance +surfaces. Columns: feature, status, oracle/diff test, upstream tests, notes. + +Status legend: ✅ done · 🟡 partial · ⬜ planned. + +## Values & typing + +| Feature | Status | Diff/unit test | Upstream tests | Notes | +|---------|--------|----------------|----------------|-------| +| Storage classes (NULL/INT/REAL/TEXT/BLOB) | ✅ | `unit/test-value` | `types*.test` | native repr + `sql-null` sentinel | +| `typeof` names | ✅ | `unit/test-value`, `diff/test-expr` | `e_expr.test` | | +| Truthiness (3-valued) | ✅ | `unit/test-value` | `e_expr.test` | TEXT/BLOB via numeric prefix | +| Numeric coercion (prefix) | ✅ | `unit/test-value` | `cast.test` | sqlite3AtoF/Atoi64 semantics | +| Numeric affinity | 🟡 | `unit/test-value`, `diff/test-select` | `affinity*.test` | insert/CAST and direct-table comparison affinity covered; broader upstream affinity corpus later | +| Comparison ordering & 3-valued logic | ✅ | `diff/test-expr`, `diff/test-select` | `e_expr.test` | NULL<number<text<blob; column-affinity conversions for `=`,`IS`,`BETWEEN`,`IN` lists, and base `CASE`; `IS TRUE/FALSE`,`AND`/`OR` | +| Arithmetic (+ - * / %), overflow→real, /0→NULL | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | | +| Bitwise (& \| << >> ~) | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | 64-bit wrap, neg-shift reverses | +| `CAST` (INT/REAL/TEXT/BLOB/NUMERIC affinity) | ✅ | `unit/test-eval`, `diff/test-expr` | `cast.test` | NUMERIC int-reduction for text/blob | +| Collations (BINARY/NOCASE/RTRIM) | ✅ | `unit/test-eval`, `unit/test-fileformat`, `diff/test-expr`, `diff/test-select`, `diff/test-constraints`, `diff/test-subquery`, `diff/test-compound` | `collate*.test` | explicit `COLLATE`; declared-column defaults; derived/compound propagation | + +## SQL surface + +| Feature | Status | Diff/unit test | Upstream tests | Notes | +|---------|--------|----------------|----------------|-------| +| Tokenizer | ✅ | `unit/test-tokenize` | tokenize-derived | full token set, SQLite keywords, hex/blob/vars | +| Parser / AST (expressions) | ✅ | `unit/test-parse` | — | Pratt parser, SQLite precedence | +| Table-less `SELECT <exprs>` / `VALUES(...)` | ✅ | `diff/test-expr`, `diff/test-select-constants` | `select1.test` | SELECT column names = source spans; VALUES column names = `columnN`; scalar/IN `VALUES` subqueries | +| Bind parameters (`?`,`?N`,`:x`,`@x`,`$x`) | ✅ | `unit/test-api`, `unit/test-parse` | `bind*.test` | SQLite numbering rules | +| `CASE` expression | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | base + searched forms | +| `BETWEEN` / `IN (list/table)` / `LIKE` / `GLOB` (+ NOT, ESCAPE) | ✅ | `diff/test-functions`, `diff/test-subquery`, `unit/test-pragma` | `like*.test`, `in*.test` | LIKE is ASCII case-insensitive by default and honors connection-local `PRAGMA case_sensitive_like`; GLOB classes; IN NULL semantics and direct-table comparison affinity; `x IN table` shorthand | +| `COLLATE` (BINARY/NOCASE/RTRIM) | ✅ | `unit/test-parse`, `unit/test-eval`, `diff/test-expr`, `diff/test-select` | `collate*.test` | comparisons, `IS` / `IS DISTINCT FROM`, `BETWEEN`, `IN` lists, ORDER/GROUP/DISTINCT, min/max | +| Scalar functions | 🟡 | `diff/test-functions`, `diff/test-date`, `unit/test-functions` | `func*.test`, `date*.test`, `timediff*.test` | typeof/abs/length/octet_length/lower/upper/coalesce/ifnull/nullif/iif/if/likely/unlikely/likelihood/concat/concat_ws/substr/substring/replace/instr/trim/ltrim/rtrim/min/max/round/sign/random/randomblob/hex/unhex/quote/unistr_quote/zeroblob/printf/format/char/unicode/unistr/sqlite_version/sqlite_source_id/changes/total_changes/last_insert_rowid/date/time/datetime/julianday/unixepoch/strftime/timediff/current_date/current_time/current_timestamp/like/glob plus SQLite math functions (`acos`, `acosh`, `asin`, `asinh`, `atan`, `atan2`, `atanh`, `ceil`, `ceiling`, `cos`, `cosh`, `degrees`, `exp`, `floor`, `ln`, `log`, `log10`, `log2`, `mod`, `pi`, `pow`, `power`, `radians`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `trunc`); REAL-to-text covers SQLite-style 15-significant-digit fixed/exponential cases; localtime/utc modifiers covered against host SQLite; broader upstream corpus later | +| `SELECT <cols/*/t.*> FROM` tables | ✅ | `diff/test-select`, `unit/test-exec` | `select*.test` | full scan; projection + expressions; aliases and parenthesized single table refs | +| `INDEXED BY` / `NOT INDEXED` table qualifiers | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-select` | `indexedby*.test` | SELECT/UPDATE-FROM table refs plus UPDATE/DELETE targets parse and validate that `INDEXED BY` names an index on the base table; SELECT table scans honor a mandatory `INDEXED BY` scan order; partial-index hints require a conservative exact-conjunct predicate proof or raise `no query solution`; simple unhinted single-table SELECTs can scan a usable user index for leading term constraints or covered projections, including expression and partial constrained searches, with `NOT INDEXED` opting out; broader cost-based selection later | +| `EXPLAIN QUERY PLAN` | 🟡 | `unit/test-parse`, `diff/test-select` | `eqp*.test` | SELECT/VALUES/WITH query wrapper returns SQLite-shaped `id,parent,notused,detail` rows for constant rows, VALUES, base-table scans, rowid primary-key searches, forced index scans/searches, covering index scans/searches, simple unhinted user-index searches, and leading composite-index constraint details; full VDBE `EXPLAIN` and complete multi-node planner trees later | +| `WHERE` filter | ✅ | `diff/test-select`, `unit/test-exec`, `unit/test-api` | `where*.test` | three-valued truthiness; direct-table comparison affinity; source-first SELECT-list alias fallback; simple single-table integer rowid/IPK equality predicates use table B-tree point lookup | +| `ORDER BY` (multi-key, ASC/DESC, ordinal, alias) | ✅ | `diff/test-select`, `unit/test-exec` | — | stable; integer and signed-integer ordinals; output aliases before source columns for bare unqualified names, source-first alias fallback inside expressions; default NULL ordering plus explicit `NULLS FIRST`/`NULLS LAST`; explicit COLLATE | +| `LIMIT` / `OFFSET` | ✅ | `diff/test-select` | — | `LIMIT a,b`; negative = unlimited | +| `DISTINCT` | ✅ | `diff/test-select` | — | numeric equality; NULLs equal; explicit COLLATE | +| Inner / cross joins + table aliases + qualified cols | ✅ | `diff/test-select` | — | nested loop; ON folded into filter; source-first SELECT-list alias fallback in ON/filter expressions | +| LEFT / RIGHT / FULL OUTER joins; `USING` / NATURAL joins | 🟡 | `diff/test-select`, `unit/test-parse` | `join*.test` | ON predicates preserve unmatched rows and can use source-first output aliases; `USING`/NATURAL hide duplicate right columns and coalesce visible output; broader upstream join corpus later | +| Aggregates (count/sum/total/avg/min/max/group_concat/string_agg, DISTINCT) | ✅ | `diff/test-aggregate`, `unit/test-aggregate`, `diff/test-functions` | `func*.test` | count(*)/count(); empty/NULL semantics; `FILTER (WHERE ...)`; aggregate-call `ORDER BY`; per-row `group_concat` separators; `string_agg(X,Y)` is the SQL-standard spelling of `group_concat`; a DISTINCT aggregate with more than one argument is rejected like SQLite | +| JSON scalar functions | 🟡 | `diff/test-json` | `json*.test` | `json` (validate + minify, preserving source string escapes like SQLite), `json_valid`, `json_type` (root or path), `json_extract` (single path returns the SQL value, multiple paths return a JSON array, `$`/`.key`/`["key"]`/`[i]`/`[#-i]` paths), `json_array_length`, `json_quote`, `json_array`, `json_object`, `json_set`/`json_insert`/`json_replace` (functional edits, `[#]` appends, out-of-range numeric index is a no-op), `json_remove`, and `json_patch` (RFC 7396 merge), plus the `->` (JSON-valued) and `->>` (SQL-valued) operators whose right operand is an integer index, a `$`-path, or a bare label; the `json_group_array` and `json_group_object` aggregates (with `FILTER`/`ORDER BY`/`DISTINCT`, `json_group_array` keeping NULLs), and the `json_each`/`json_tree` table-valued functions (see the table-valued-functions row); an RFC-8259 parser/minifier backs them and the oracle verifies output byte-for-byte. A value returned by a JSON function carries SQLite's JSON subtype (tracked per statement by result-string identity), so `json_array(json_object(...))`, `json_set(x,p,json(...))`, and `json_extract` of a container embed as JSON rather than double-encoding, while plain strings stay quoted; `subtype(X)` reports `74` for a JSON-subtyped text value and `0` otherwise. The parsing functions accept **JSON5** input (comments, unquoted/single-quoted keys and strings, trailing commas, hex and `.5`/`5.` numbers, `+`/`Infinity`/`NaN`), output stays canonical, and `json_valid` honors the strict (default) vs JSON5 (`0x02`) flag bit. The **JSONB** binary format is supported: a byte-for-byte-compatible encoder/decoder backs `jsonb`/`jsonb_extract`/`jsonb_array`/`jsonb_object`/`jsonb_set`/`jsonb_insert`/`jsonb_replace`/`jsonb_remove`/`jsonb_patch` (returning JSONB blobs with the JSON subtype), every `json_*` function reads a JSONB blob argument, the `jsonb_group_array`/`jsonb_group_object` aggregates produce JSONB, and `hex(jsonb(...))` matches upstream exactly | +| `GROUP BY` / `HAVING` | ✅ | `diff/test-aggregate`, `diff/test-select`, `unit/test-aggregate` | — | aggregates inside expressions; GROUP BY ordinals and source-first alias fallback; HAVING output aliases; explicit COLLATE grouping | +| Compound `UNION`/`UNION ALL`/`INTERSECT`/`EXCEPT` | ✅ | `diff/test-compound`, `unit/test-compound` | `select4*.test` | trailing ORDER BY/LIMIT with ordinal, alias, and output-expression resolution; numeric dedupe; SELECT and VALUES arms | +| Subqueries: scalar, `EXISTS`, `IN (SELECT)` (+ correlation) | ✅ | `diff/test-subquery`, `unit/test-subquery` | `subquery*.test` | lexically-scoped column resolution; `IN (SELECT)` uses output affinity for single-column comparisons | +| Subqueries in FROM (derived tables) | ✅ | `diff/test-subquery`, `unit/test-subquery` | `subquery*.test` | SELECT/compound/VALUES derived tables preserve output collation and affinity; no synthetic rowid | +| `WITH` common table expressions | 🟡 | `unit/test-parse`, `unit/test-subquery`, `unit/test-write`, `diff/test-select`, `diff/test-write`, `diff/test-returning`, `diff/test-select-constants` | `with*.test` | SELECT/VALUES CTEs, including column aliases, compound and nested WITH CTE bodies, derived-table WITH queries, and `WITH RECURSIVE` anchors plus one or more recursive arms under `UNION`/`UNION ALL`, with queue `ORDER BY`/`LIMIT`/`OFFSET`; `WITH` on INSERT, UPDATE, and DELETE; broader recursive edge cases later | +| Window functions | 🟡 | `unit/test-parse`, `diff/test-functions` | `window*.test` | inline or named `OVER` clauses with default frames for built-ins (`row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`, `lead`, `lag`, `first_value`, `last_value`, `nth_value`) and aggregate windows (`count`, `sum`, `total`, `avg`, `min`, `max`, `group_concat`) including aggregate `FILTER (WHERE ...)`; named base-window inheritance for `WINDOW` clauses and inline `OVER (base ...)`; explicit `ROWS`, `GROUPS`, and single-term `RANGE` frame clauses with `EXCLUDE` variants for aggregate and value windows; window functions over grouped aggregate result rows; `FILTER` on non-aggregate built-ins and `DISTINCT` window aggregates are rejected like SQLite; broader upstream window corpus later | +| `CREATE VIEW` / `DROP VIEW`; selecting from views | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-select`, `diff/test-write` | `view*.test` | schema-only views with optional output column list; SELECT/WITH/VALUES view bodies; circular view reads are rejected; INSTEAD OF triggers can route view INSERT/UPDATE/DELETE; `CREATE TEMP VIEW` lives in the temp schema (listed in `sqlite_temp_master`, never persisted) and its body may read temp or main tables | +| `CREATE TABLE` / `CREATE TABLE ... AS SELECT` (+ IF NOT EXISTS) | 🟡 | `diff/test-write`, `diff/test-strict`, `unit/test-fileformat`, `unit/test-write` | `createtab*.test`, `strict*.test` | rowid tables; built-in `main.` qualifier; duplicate column names rejected; AUTOINCREMENT/sqlite_sequence; CTAS materializes SELECT/WITH/VALUES output and generated column names; declared collations captured; `STRICT` rowid tables enforce storage classes and preserve `ANY`; STORED and VIRTUAL generated columns are dependency-ordered on INSERT/UPDATE, virtual columns are omitted from physical records, generated-column indexes/UNIQUE constraints are maintained, and upstream readback/integrity_check passes; `WITHOUT ROWID` is supported (see its own row); TEMP/virtual tables are explicit unsupported cases | +| WITHOUT ROWID tables | 🟡 | `unit/test-without-rowid`, `diff/test-without-rowid` | `withoutrowid*.test` | stored as a PRIMARY-KEY-keyed index B-tree (PK columns first, then the remaining columns); single/composite PKs with per-column `ASC`/`DESC` and `COLLATE`; PK columns are implicitly NOT NULL; duplicate-PK raises `SQLITE_CONSTRAINT_PRIMARYKEY`/`_UNIQUE`; `rowid`/`oid`/`_rowid_` are not columns; secondary and UNIQUE indexes are keyed by the PK (not a rowid); INSERT/SELECT/UPDATE/DELETE plus upstream `integrity_check` and bidirectional readback round-trip pass; missing PRIMARY KEY and AUTOINCREMENT are rejected like upstream; the rowid index planner does not apply, so reads full-scan | +| TEMP tables / indexes / views (in-memory `temp` schema) | 🟡 | `unit/test-temp`, `unit/test-parse` | `temp*.test` | `CREATE TEMP TABLE/INDEX/VIEW` live in a separate in-memory image that shadows `main` for unqualified names and never persists to disk; full INSERT/SELECT/UPDATE/DELETE/DROP, secondary indexes on temp tables, and same-name shadowing; a plain `CREATE INDEX` lands wherever its target table lives, and `CREATE TABLE temp.x` / `DROP TABLE temp.x` route to the temp schema; unqualified `sqlite_master` reports `main` while `sqlite_temp_master` / `sqlite_temp_schema` report the temp schema (empty until a temp object exists, and not writable); `PRAGMA database_list` reports `temp` once a temp object exists; `temp.`-qualified table and 3-part column references resolve to the temp schema, a single statement may join temp and main tables, and BEGIN/COMMIT/ROLLBACK/SAVEPOINT cover temp and main together (one snapshot spans both); a 3-part column ref (`main.both.b` vs `temp.both.b`) disambiguates two same-named tables joined from different schemas, while the bare `both.b` still reaches its table and is reported ambiguous when both schemas are joined | +| ATTACH / DETACH | 🟡 | `unit/test-attach`, `diff/test-attach`, `diff/test-crossdb` | `attach*.test` | `ATTACH '<file>'` or `':memory:' AS <name>` adds a named database (searched after temp/main for unqualified names) and `DETACH` removes it; `CREATE TABLE <db>.t` directs a new table into a database; file-backed attachments persist on write and pass upstream `integrity_check`/readback; `PRAGMA database_list` reports them at seq 2+. Reads resolve each table reference to its database via a per-query catalog, so a single SELECT/join/subquery can read across main, temp, and attached databases (verified against upstream in `diff/test-crossdb`). Each write (INSERT/UPDATE/DELETE/DDL) targets one database, but its source may read others: `INSERT INTO aux.t SELECT … FROM main.u`, `UPDATE … FROM <other db>`, cross-database subqueries, and `CREATE TABLE aux.x AS SELECT … FROM main.y` all work (verified vs upstream). Qualified column references resolve in one, two, or three parts (`col`, `t.col`, `db.t.col`). Transactions span databases: `BEGIN`/`SAVEPOINT` snapshot main, temp, and every attached database, so `ROLLBACK`/`ROLLBACK TO` atomically undo all of them and `COMMIT` persists all file-backed databases together | +| Virtual tables | ⬜ | `unit/test-parse`, `diff/test-write` | `vtab*.test` | `CREATE VIRTUAL TABLE` is an explicit unsupported case; FTS/RTree/etc. remain out of scope for the current storage engine | +| Table-valued functions in FROM | 🟡 | `unit/test-tvf`, `diff/test-json` | `tabfunc*.test`, `json*.test` | `name(args)` in FROM parses to a tvf-ref that resolves to a synthetic scan; `json_each(X[,path])` and `json_tree(X[,path])` (8 SQLite columns; `key`/`value`/`type`/`atom`/`parent`/`fullkey`/`path` match the oracle, `id`/`parent` are a best-effort sequential counter since SQLite's reflect JSONB byte offsets — `json_tree` pre-order DFS with the path-scoped root keeping its key/parent path) `generate_series(start[,stop[,step]])`, and the schema-introspection table-valued PRAGMA functions (`pragma_table_info`/`pragma_table_xinfo`/`pragma_index_list`/`pragma_index_info`/`pragma_index_xinfo`/`pragma_foreign_key_list`, e.g. `SELECT * FROM pragma_table_info('t')`) are implemented; a tvf may be **lateral/correlated** — `SELECT t.id, je.value FROM t, json_each(t.data) je` re-evaluates the function against each outer row (cross/inner joins; outer joins fall back to a single non-correlated evaluation); user/eponymous virtual-table tvfs are not done. Non-reserved keywords (`key`, `value`, `offset`, `match`, ...) are now accepted as column names, matching SQLite's identifier fallback | +| Triggers | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write` | `trigger*.test` | executable subset: schema-stored row-level `BEFORE`/`AFTER INSERT`, `BEFORE`/`AFTER UPDATE [OF ...]`, and `BEFORE`/`AFTER DELETE` triggers on ordinary tables; `INSTEAD OF INSERT`/`UPDATE [OF ...]`/`DELETE` triggers on views; optional `WHEN`, trigger-body INSERT/UPDATE/DELETE/SELECT steps, `NEW`/`OLD` row references, `RAISE(ABORT/FAIL/ROLLBACK, expr)` with trigger constraint errors, `RAISE(IGNORE)` trigger-program/row skipping, and `DROP TRIGGER`; `CREATE TEMP TRIGGER` fires on a temp table or on a table in another schema (a temp trigger on a main/attached table is created and fired via the cross-schema object set), and each trigger-body write step is routed to the schema that holds its target table, so a temp trigger on a main table can log into a temp table; not yet: statement triggers, recursive-compatibility edge cases, or the full trigger-program grammar | +| `DROP TABLE` (+ IF EXISTS) | 🟡 | `diff/test-conflict`, `unit/test-conflict`, `unit/test-constraints`, `diff/test-constraints` | `drop*.test` | rewrites schema; dropped table/index pages go to freelist; with foreign keys enabled, runs SQLite-like implicit DELETE actions for valid incoming references | +| `CREATE INDEX` / `CREATE UNIQUE INDEX` (+ IF NOT EXISTS) | 🟡 | `unit/test-parse`, `unit/test-write`, `unit/test-constraints`, `diff/test-write`, `diff/test-constraints` | `index*.test` | rowid-table column and expression indexes; built-in `main.` qualifier on index name; ASC/DESC and COLLATE; partial-index WHERE predicates; UNIQUE enforced; planner use later | +| `DROP INDEX` (+ IF EXISTS) | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write`, `diff/test-analyze` | `drop*.test` | frees user-index pages; removes matching `sqlite_stat1` rows; upstream-readable file coverage; autoindexes rejected | +| `REINDEX` | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write` | `reindex*.test` | rebuilds all indexes, a named table/index, or indexes using built-in BINARY/NOCASE/RTRIM collations; custom collations later | +| `INSERT ... VALUES` / `DEFAULT VALUES` / `INSERT ... SELECT` (+ insert affinity) | ✅ | `diff/test-write`, `unit/test-write` | `insert*.test` | multi-page; SELECT source materialized before insert; AUTOINCREMENT sequence allocation | +| `INSERT OR IGNORE / OR REPLACE / OR FAIL / OR ROLLBACK`, `REPLACE INTO` | ✅ | `diff/test-conflict`, `unit/test-conflict` | `conflict*.test` | ignore skips violations; replace removes conflicts; FAIL preserves prior row changes in the statement; ROLLBACK aborts the statement in autocommit and rolls back the active transaction; schema-level `ON CONFLICT IGNORE/REPLACE/FAIL/ROLLBACK` policies for rowid/UNIQUE/NOT NULL, including NOT NULL defaults for REPLACE | +| UPSERT (`ON CONFLICT ... DO NOTHING/UPDATE`) | 🟡 | `diff/test-conflict`, `diff/test-returning`, `unit/test-conflict`, `unit/test-parse` | `upsert*.test` | column conflict targets over rowid/UNIQUE constraints, including partial UNIQUE index targets with matching WHERE predicates; `excluded` values, optional DO UPDATE WHERE, and multi-clause target selection; broader edge cases later | +| `UPDATE ... SET ... [FROM] [WHERE]` | ✅ | `diff/test-write`, `diff/test-conflict`, `diff/test-returning`, `unit/test-write`, `unit/test-conflict` | `update*.test` | multi-page rewrite; `UPDATE ... FROM` over tables/joins/CTEs; `OR IGNORE` / `OR REPLACE` / `OR FAIL` / `OR ROLLBACK` for rowid/UNIQUE/NOT NULL/CHECK conflicts; `ORDER BY` + `LIMIT/OFFSET` target selection | +| `DELETE FROM ... [WHERE]` | ✅ | `diff/test-write`, `diff/test-returning`, `unit/test-write` | `delete*.test` | multi-page rewrite; `ORDER BY` + `LIMIT/OFFSET` target selection | +| DML `RETURNING` | 🟡 | `diff/test-returning`, `unit/test-api`, `unit/test-conflict`, `unit/test-parse` | `returning*.test` | `INSERT`/`UPDATE`/`DELETE RETURNING` over target rows, including `*`, `t.*`, scalar expressions, UPSERT affected rows, and UPDATE/DELETE target `ORDER BY` + `LIMIT/OFFSET`; broader edge cases later | +| `ALTER TABLE` | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write`, `diff/test-constraints` | `alter*.test` | `ALTER TABLE ... RENAME TO ...` rewrites table/index schema rows and sqlite_sequence entries; `ADD [COLUMN]` supports ordinary columns with defaults, validates added CHECK constraints, and applies SQLite's REFERENCES/default rule when foreign keys are enabled, while rejecting PK/UNIQUE/generated additions; `RENAME [COLUMN] ... TO ...` rewrites table constraints plus user index column, expression, and partial-predicate references; `DROP [COLUMN]` rewrites ordinary non-dependent columns and rejects indexed/partial-index predicate dependencies; broader dependency rewrites later | +| `NOT NULL` constraint | ✅ | `unit/test-write` | `notnull*.test` | on INSERT + UPDATE; exact errmsg | +| `DEFAULT` column values | ✅ | `diff/test-write`, `unit/test-write` | `default*.test` | literal defaults; affinity applied | +| `UNIQUE` / `PRIMARY KEY` (column, composite, text, integer) | ✅ | `diff/test-constraints`, `unit/test-constraints` | `unique*.test` | enforced by row scan; declared collations honored; autoindexes maintained for integrity_check; NULLs distinct | +| `CHECK` constraints (column + table level) | ✅ | `unit/test-check` | `check*.test` | fails only when expr is false; enforced on INSERT/UPDATE; OR IGNORE skips | +| FOREIGN KEY | 🟡 | `unit/test-constraints`, `diff/test-constraints` | `fkey*.test` | `PRAGMA foreign_keys` and `defer_foreign_keys`; immediate and `DEFERRABLE INITIALLY DEFERRED` timing; pre-existing violations are tolerated until touched; parent-key mismatch validation for referenced PK/UNIQUE keys, including partial-index and collation eligibility; `ON DELETE`/`ON UPDATE`/`DROP TABLE` `RESTRICT`/`CASCADE`/`SET NULL`/`SET DEFAULT`; actions are gated by `PRAGMA foreign_keys`; deeper edge cases later | +| `BEGIN` / `COMMIT` / `END` / `ROLLBACK` / `SAVEPOINT` / `RELEASE` / `ROLLBACK TO` | 🟡 | `unit/test-transaction`, `unit/test-attach`, `diff/test-transaction` | `trans*.test`, `savepoint*.test` | nested image snapshots spanning main, temp, and every attached database (cross-database atomic ROLLBACK/COMMIT); deferred file write until COMMIT/outermost RELEASE, then all file-backed databases persist together; DELETE-mode rollback journal with hot-journal recovery; locking/fsync later | +| On-disk index B-trees | 🟡 | `diff/test-conflict`, `diff/test-write`, `diff/test-select` | — | autoindex + user-created roots maintained through DML; low-level index scans read leaf/interior pages and overflow payloads; multi-level and overflow index writes pass upstream integrity_check; mandatory, simple unhinted, and covered-projection SELECT scans use user-index order; broader planner use later | +| Freelist management | 🟡 | `diff/test-write`, `diff/test-bigtable`, `diff/test-conflict` | — | table rebuild/drop pages are freed and allocator reuses freelist pages | +| `SELECT ... FROM sqlite_master/sqlite_schema` | ✅ | `unit/test-write` | — | implicit schema table | +| PRAGMA (table_info / table_xinfo / table_list / index_list / index_info / index_xinfo / database_list / collation_list / foreign_key_list / foreign_key_check / user_version / application_id / schema_version / data_version / journal_mode / page_count / freelist_count / page_size / cache_size / integrity_check / quick_check / encoding / foreign_keys / defer_foreign_keys / recursive_triggers / trusted_schema / ignore_check_constraints / query_only / count_changes / reverse_unordered_selects / read_uncommitted / automatic_index / busy_timeout / synchronous / temp_store / locking_mode / case_sensitive_like / secure_delete / cell_size_check / fullfsync / checkpoint_fullfsync / journal_size_limit / analysis_limit / optimize / mmap_size / wal_autocheckpoint / threads / function_list / pragma_list / module_list) | 🟡 | `diff/test-pragma`, `unit/test-pragma`, `unit/test-check`, `diff/test-constraints` | `pragma*.test` | introspection + a few settings; `function_list` lists jsqlite's built-in functions (type `s`/`w`), `pragma_list` its PRAGMA names, and `module_list` its eponymous table-valued functions (`json_each`/`json_tree`/`generate_series`) — content is jsqlite-specific (SQLite's per-build `narg`/`flags` are not modelled), so these are unit-tested, and all three are also usable as the `pragma_*` table-valued form; schema-qualified introspection (`PRAGMA temp.table_info`, `PRAGMA <db>.index_list`, ...) resolves the named schema, and unqualified introspection of a table name shadows temp over main then attached; `table_info`/`table_xinfo` cover tables and SELECT/WITH/VALUES/compound views; `table_list` reports main-schema tables/views and built-in schema rows; `index_info`/`index_xinfo` include expression-index rows plus rowid auxiliary columns for rowid-table indexes; `database_list` reports `main`, then `temp` once a temp object exists, then each attached database; `collation_list` reports built-in collations; page/freelist counts read the database image/header; user_version/application_id/schema_version persist; `recursive_triggers`, `trusted_schema`, `ignore_check_constraints`, `query_only`, `count_changes`, `reverse_unordered_selects`, `read_uncommitted`, `automatic_index`, `busy_timeout`, `synchronous`, `temp_store`, `locking_mode`, `case_sensitive_like`, `secure_delete`, `cell_size_check`, `fullfsync`, `checkpoint_fullfsync`, `journal_size_limit`, `analysis_limit`, `mmap_size`, `wal_autocheckpoint`, and `threads` are connection-level flags/settings; no-row `PRAGMA optimize` is accepted with SQLite-shaped columns, but recommendation output such as `optimize(-1)` is not modeled yet; `automatic_index` does not disable ordinary user indexes, matching SQLite; transient automatic indexes are not modeled yet; `query_only` rejects writes with `SQLITE_READONLY`; `count_changes` returns SQLite-shaped affected-row counts for INSERT/UPDATE/DELETE without RETURNING, including view row counts for INSTEAD OF trigger DML while `changes()` remains zero; `reverse_unordered_selects` reverses simple unordered base-table SELECT scans while ORDER BY and VALUES remain stable; `case_sensitive_like` changes LIKE operator/function matching for reads and DML constraint evaluation; `ignore_check_constraints` gates CHECK enforcement and integrity CHECK reporting; `data_version` covers single-connection shape, not cross-connection invalidation; journal_mode/cache_size/synchronous/locking/fullfsync/WAL/checkpoint settings do not yet model full pager locking/fsync/WAL behavior | +| `VACUUM` | 🟡 | `diff/test-vacuum`, `unit/test-parse`, `unit/test-transaction` | `vacuum*.test` | simple `VACUUM [schema]` and `VACUUM INTO` database copies; rejects transaction-time VACUUM and existing output files; physical compaction later | +| `ANALYZE` | 🟡 | `diff/test-analyze`, `unit/test-parse`, `unit/test-transaction` | `analyze*.test` | accepts `ANALYZE`, schema targets, table/index targets, SQLite-like unknown-name errors, and `sqlite_stat1` rows for tables plus ordinary, expression, and partial indexes; broader planner statistics later | +| REAL affinity on read | ✅ | `diff/test-select`, `unit/test-exec` | — | integer-stored reals → REAL | +| `sqlite_schema` read + CREATE TABLE column resolution | ✅ | `unit/test-fileformat` | — | column names + INTEGER PRIMARY KEY rowid alias | + +## API + +| Feature | Status | Diff/unit test | Notes | +|---------|--------|----------------|-------| +| open/open-v2/close | ✅ | `unit/test-api`, `diff/*` | in-memory, ordinary read-write/create files, and read-only file handles | +| prepare/step/reset/finalize | ✅ | `unit/test-api` | lifecycle: before→at-row→done | +| column count/name/type/value | ✅ | `unit/test-api` | | +| bind_* / clear-bindings / bind-parameter-count/name/index | ✅ | `unit/test-api` | bind by index; range-checked; generic and typed helpers; named parameter metadata lookup | +| exec/query helpers | ✅ | `unit/test-api` | step-and-finalize convenience helpers with positional bind args | +| changes / total_changes / last_insert_rowid | ✅ | `unit/test-api` | row-change counters and last inserted rowid | +| errcode / extended_errcode / errmsg | ✅ | `unit/test-api` | connection error state for raised SQLite errors | +| Result/error codes | ✅ | `unit/test-constants`, `unit/test-error` | primary + extended codes | +| Application-defined functions / collations | 🟡 | `unit/test-api` | `sqlite-create-function` registers a scalar SQL function (fixed arity or `-1` variadic; same name+arity replaces); `sqlite-create-aggregate` registers an aggregate (init/step/final), and `sqlite-create-collation` registers a collating sequence usable in `COLLATE`, `ORDER BY`, and column definitions. Per-connection, consulted at prepare (aggregate detection) and execute. the commit/rollback/update hooks (`sqlite-commit-hook`/`sqlite-rollback-hook`/`sqlite-update-hook`; the commit hook can veto a COMMIT into a ROLLBACK; the update hook fires per base-table row change with op/db/table/rowid, including trigger-internal changes). Incremental BLOB I/O (`sqlite-blob-open`/`-read`/`-write`/`-bytes`/`-reopen`/`-close`) addresses bytes within one (schema, table, column, rowid) cell: reads return a sub-range, writes overwrite in place without changing the value's length and persist through a row update, read-only handles refuse writes, and out-of-range access errors. A progress handler (`sqlite-progress-handler` db n proc) is invoked once per `n` scanned B-tree rows and aborts the statement with `SQLITE_INTERRUPT` if it returns true, and `sqlite-interrupt` sets a one-shot flag that aborts the running/next statement the same way. An authorizer (`sqlite-set-authorizer` db proc) is consulted at prepare for each action a statement performs (`SQLITE_SELECT`/`INSERT`/`UPDATE`/`DELETE`/`CREATE_*`/`DROP_*`/`PRAGMA`/`ATTACH`/`DETACH`/`ALTER_TABLE`/`REINDEX`/`ANALYZE`, with table/column/name args); `SQLITE_DENY` rejects the statement with `SQLITE_AUTH`, `SQLITE_OK` allows. Statement-level granularity (column-level `SQLITE_READ` and `SQLITE_IGNORE`-as-NULL are not modelled, and TEMP/attached schemas report as `main`). `sqlite-create-window-function` registers an aggregate usable in an `OVER` clause; jsqlite recomputes the aggregate over each frame, so a user aggregate (or a window function) matches the built-in window aggregates exactly across partitions and sliding frames (the `xInverse`/`xValue` callbacks are accepted but not required). `sqlite-create-module` registers a read-only eponymous virtual-table module: `SELECT ... FROM name(args)` evaluates the args and calls the module's rows function, and the result participates in projection, `WHERE`, aggregation, and joins like any table-valued function. Not yet on the virtual-table side: `CREATE VIRTUAL TABLE` persistence, constraint pushdown (`xBestIndex`), and writable modules. A **C ABI shim** (`src/jsqlite/cshim.ss` + `shim/jsqlite.h`) exposes the core operations as C-callable function pointers — `open`/`prepare`/`step`/`column_count`/`column_int`/`column_text`/`column_type`/`finalize`/`close`/`errmsg`, with connections/statements as opaque integer handles — gathered into a `jsq_api` struct by `build-c-api`; `shim/driver.c` is a C program that drives a full CREATE/INSERT/SELECT cycle through it (`make cshim`), and `unit/test-api` calls the same entry points with the C calling convention. The Chez runtime hosts the engine, so the C side obtains the `jsq_api` from the embedding rather than from a standalone `main` that boots Chez itself. `sqlite-backup` copies a whole database (snapshot, or load a file into memory) | + +## File format + +| Feature | Status | Test | Notes | +|---------|--------|------|-------| +| Header parse (100 bytes) | ✅ | `unit/test-fileformat` | page sizes 512–65536, reserved bytes | +| Varint / record decode | ✅ | `unit/test-record` | serial types, signed ints, float64, text/blob | +| Varint / record encode | ✅ | `unit/test-record` | round-trips; int64 extremes | +| Table B-tree read (interior + leaf) | ✅ | `unit/test-fileformat`, `diff/test-fileread` | full scan, rowid order, rowid point lookup | +| Overflow pages (read) | ✅ | `diff/test-fileread` | exact src/btree.c local-payload math | +| Create DB / CREATE TABLE / INSERT (write) | ✅ | `diff/test-write`, `diff/test-bigtable` | multi-page; verified by upstream integrity_check | +| Multi-page B-tree write (interior + leaves) | ✅ | `diff/test-bigtable`, `unit/test-write` | bottom-up rebuild; 1500+ rows | +| Overflow on write | ✅ | `diff/test-bigtable` | large payloads spill to overflow chain | +| Reserved-bytes pages (write) | ✅ | `diff/test-reserved` | B-tree cell layout, capacities and overflow thresholds honor the per-page reserved-bytes count, so jsqlite writes valid pages to upstream-created databases (verified by upstream integrity_check) | +| Text encoding (UTF-8 / UTF-16) | ✅ | `diff/test-utf16` | reads and writes UTF-8, UTF-16LE, and UTF-16BE databases: records serialize text in the database's declared encoding, and BINARY collation orders text by its encoded bytes (so UTF-16 text indexes and ORDER BY match upstream). `PRAGMA encoding` reports the database's actual encoding, and `PRAGMA encoding='UTF-16le'`/`'UTF-16be'`/`'UTF-8'` on an empty database sets it, so jsqlite can create a UTF-16 database from scratch (upstream-verified) | +| Rollback journal (crash safety) | 🟡 | `diff/test-transaction` | simple DELETE-mode journal and hot-journal restore; no fsync/locking model yet | +| Index B-tree read/write | 🟡 | `diff/test-conflict`, `diff/test-write`, `diff/test-select` | reads leaf/interior index pages, including overflow payloads; autoindex and user-created column/expression roots; multi-level and overflow index writes; mandatory `INDEXED BY`, simple unhinted, and covered-projection SELECT scans can use index order; broader planner selection later | +| WAL (read + write + checkpoint) | 🟡 | `unit/test-wal`, `unit/test-api` | jsqlite reads a database left in WAL mode by merging committed `-wal` frames over the main image when the file is opened: frames are checksum-validated (algorithm verified against upstream output), the latest committed frame wins per page, the database is resized to the commit, and torn/uncommitted frames are ignored. `PRAGMA journal_mode=WAL` switches the connection to WAL mode, after which a commit **appends** the pages it changed as new frames to the `-wal` (continuing the salt and running-checksum chain) rather than rewriting the whole database; the main file is left untouched, and any reader (jsqlite or, after a checkpoint, upstream SQLite) sees the merged state. Because commits append, several jsqlite processes writing one WAL-mode database under the write lock accumulate correctly (the 4-process stress test passes in WAL mode too). The `-wal` is folded back into the main file (checkpoint) when the connection leaves WAL mode or closes — under the write lock, so it cannot race a concurrent writer — leaving an upstream-readable rollback-mode database (`integrity_check` passes). A **shared-memory WAL-index** (`src/jsqlite/shm.ss`) backs WAL coordination: every connection `mmap`s the `<path>-shm` file `MAP_SHARED` (via libc, not libsqlite3), and a writer publishes the WAL salts, committed frame count, page count, a per-commit **generation** number, and a frame→page index into it under the write lock. Other processes read the generation from shared memory to decide when to reload (instead of re-scanning the `-wal`); a checkpoint invalidates the index in place (the file is kept so concurrent mappings observe the change). The layout is jsqlite's own (not byte-compatible with upstream's wal-index), so a *live, uncheckpointed* `-wal` is still not exposed to a concurrent **upstream** process; that, and per-reader snapshots, are the remaining WAL items | +| Multi-process write locking | 🟡 | `unit/test-api`, `tools/concurrency-test` (`make concurrency`) | A writer takes a POSIX `flock` advisory exclusive lock (on a `<path>-lock` sidecar, via libc — not libsqlite3) for its read-modify-write: an autocommit statement locks per statement, an explicit transaction from `BEGIN`/first `SAVEPOINT` through `COMMIT`/`ROLLBACK`. On taking the lock it refreshes the main image to the latest committed state (detected by the header change counter) so writes build on, rather than clobber, concurrent commits. A contended lock retries under `sqlite-busy-timeout` / `sqlite-busy-handler` and otherwise returns `SQLITE_BUSY`. Opening a database (hot-journal recovery + initial load) also holds the lock, so a starting connection cannot mistake another process's in-progress rollback journal for a crash and undo its commit. A 4-process stress test inserts concurrently in both rollback and WAL journal modes with no lost updates, no rowid collisions, and a passing `integrity_check` (`make concurrency`). Locking coordinates jsqlite processes with each other (advisory, keyed on the file); reads use their open-time snapshot (no shared-lock/`-shm` reader-snapshot coordination yet), and attached databases are not separately locked | + +## Known limitations + +- Virtual tables, full trigger support, broader window functions, native WAL + writing (producing `-wal` frames), and index planner use remain planned or + partial as noted above. WAL-mode databases are read (frames merged on open) + and writable: the first jsqlite write checkpoints the merge into the main file + (rollback mode) and removes the `-wal`. +- The write path honors each page's reserved-bytes count, so jsqlite can write + to databases created by upstream SQLite (which currently reserves bytes per + page), not only to databases it created itself. +- TEMP tables/indexes/views and ATTACH/DETACH are supported, including reads that + span databases in one statement (cross-database joins/subqueries), writes whose + source reads another database (INSERT…SELECT, UPDATE…FROM, CTAS), and + transactions that span databases (BEGIN/SAVEPOINT snapshot main, temp, and every + attached database; ROLLBACK/COMMIT are atomic across all of them). +- WITHOUT ROWID tables are supported (PK-keyed index-B-tree storage, file-format + compatible) but always full-scan, as the rowid index planner does not apply. new file mode 100644 --- /dev/null +++ b/vendor/jsqlite/docs/test-plan.md @@ -0,0 +1,52 @@ +# Test plan + +Compliance is tracked on three surfaces, each with tests: + +1. **SQL behavior** — same rows, column names, storage classes, errors, side + effects, and PRAGMA/transaction behavior as SQLite 3.54.0. +2. **File format** — databases written by jsqlite open in upstream SQLite and + pass `PRAGMA integrity_check`; upstream databases open and query in jsqlite. +3. **API shape** — `sqlite3_*`-shaped lifecycle and result codes. +