build: fetch jerboa-sqlite from Forgejo
ober
c9aac55fe3b1cfa373e0013f66bfcdcee9fe45f7
--- a/.gitignore +++ b/.gitignore @@ -80,6 +80,9 @@ new.txt # External project stubs (belong in their own repos) lib/gerbil-litehtml/ +# Revision-pinned build dependency fetched from Forgejo by `make vendor-deps`. +/vendor/jsqlite/ + # VM test artifacts (large images, keys, ISOs) tests/vm/*.qcow2 tests/vm/*.qcow2.xz --- a/Makefile +++ b/Makefile @@ -6,8 +6,9 @@ export SOURCE_DATE_EPOCH HOST_UNAME_S := $(shell uname -s) HOST_UNAME_M := $(shell uname -m) HOST_UNAME_O := $(shell uname -o 2>/dev/null || true) -# SQLite is provided by vendored jsqlite. Packet capture remains explicit -# opt-in because rscap 0.3 does not compile with the supported Rust toolchain. +# SQLite is fetched from the revision-pinned jerboa-sqlite Forgejo repository. +# Packet capture remains explicit opt-in because rscap 0.3 does not compile +# with the supported Rust toolchain. JERBOA_NATIVE_FEATURES ?= full STATIC ?= 0 STATIC_ENABLED := $(filter 1 yes true on,$(STATIC)) @@ -308,6 +309,8 @@ help: @echo "Build:" @echo " chez Build and install vendored Chez Scheme locally" @echo " build Compile all Jerboa libraries" + @echo " vendor-deps Fetch and verify pinned Forgejo dependencies" + @echo " vendor-clean Remove fetched dependencies" @echo " binary Build a self-contained jerboa-bin binary (FreeBSD/Linux/macOS)" @echo " chez-cross Build cross Chez (target libkernel.a + boot files + xpatch)" @echo " Args: CHEZ_TARGET_MACHINE=<mt> CROSS_CC=<cross-cc>" @@ -544,7 +547,17 @@ $(SCHEME): vendor/ChezScheme/configure $(MAKE) -C $(CHEZ_BUILD_DIR) install test -x $(SCHEME) -build: chez transpile +.PHONY: vendor-deps vendor-clean +vendor-deps: vendor/jsqlite + +vendor/jsqlite: vendor-lock.env support/fetch-vendor.sh + @echo "=== Fetching locked jerboa-sqlite source ===" + @sh support/fetch-vendor.sh jsqlite + +vendor-clean: + rm -rf "$(JERBOA_HOME)/vendor/jsqlite" + +build: vendor-deps chez transpile @mkdir -p build JERBOA_BUILD_GENSYM_PREFIX_FILE=$(CURDIR)/build/jerboa-build-gensym-prefix.txt \ $(SCHEME) --libdirs $(LIBDIRS) --script support/build.ss @@ -1238,7 +1251,7 @@ MCP_EXTERNAL_TEST = mcp/test/external-lib-test.ss # Source hygiene scoped to the MCP tree. Jerboa proper legitimately uses .sls # and (chezscheme); only mcp/*.ss must stay pure Jerboa and data/ stays .sexp. -mcp-check: +mcp-check: vendor-deps @test "$$(find mcp -name '*.sls' | wc -l | tr -d ' ')" = "0" \ || { echo "ERROR: .sls under mcp/ — server must stay pure Jerboa" >&2; exit 1; } @test "$$(find data -name '*.json' | wc -l | tr -d ' ')" = "0" \ @@ -1368,7 +1381,7 @@ api-signatures: chez @JERBOA_HOME=$(JERBOA_HOME) $(SCHEME) --script $(API_SIG_GEN) # Regenerate the completion tables from the live API map (data/api-signatures.sexp). -lsp-gen: chez +lsp-gen: vendor-deps chez @JERBOA_HOME=$(JERBOA_HOME) $(SCHEME) --libdirs $(LIBDIRS) --script $(LSP_GEN) # Compile-check the whole server from source (imports the full cascade). @@ -2911,7 +2924,7 @@ audit-native: jpkg-audit: build @$(SCHEME) --libdirs $(LIBDIRS) --script tools/jpkg-main.ss audit -vendor-jsqlite-security-check: +vendor-jsqlite-security-check: vendor-deps @support/check-vendor-jsqlite-security.sh native-export-review-check: @@ -2921,7 +2934,7 @@ tcb-drift-check: @set -eu; \ if git diff --quiet -- .jerboa-system support/container-dependencies.lock \ support/*.c support/*.h jerboa-native-rs/Cargo.lock browser-repl/Cargo.lock \ - vendor/jsqlite; then \ + vendor-lock.env support/fetch-vendor.sh; then \ echo "tcb-drift: OK"; \ elif git diff --quiet -- data/changelog.sexp; then \ echo "tcb-drift: TCB-affecting files changed; add a data/changelog.sexp note" >&2; \ @@ -2933,15 +2946,15 @@ tcb-drift-check: security-audit: @tools/security-audit.sh -source-balance: +source-balance: vendor-deps @git ls-files '*.ss' '*.sls' \ | grep -v '^vendor/' \ | xargs $(SOURCE_BALANCE_RUNNER) support/check-source-balance.ss -restrict-closure-check: +restrict-closure-check: vendor-deps @$(SCHEME) --libdirs $(LIBDIRS) --script tools/check-restrict-closure.ss -unsafe-deserialize-check: +unsafe-deserialize-check: vendor-deps @git ls-files '*.ss' '*.sls' \ | grep -E '^(lib|src|tools|mcp|lsp)/' \ | grep -v '^vendor/' \ --- a/data/changelog.sexp +++ b/data/changelog.sexp @@ -1,8 +1,20 @@ (("description" . - "Machine-readable changelog of Jerboa API drift. Consumers (LLM tooling, lints, jerboa_verify) use this to invalidate stale recommendations and to suggest migrations when a symbol is renamed or relocated.") + "Machine-readable changelog of Jerboa API drift. Consumers (LLM tooling, lints, jerboa_verify) use this to invalidate stale recommendations and to suggest migrations when a symbol is renamed or relocated.") ("entries" - (("added" ("bytevector-u8-set!" "bytevector-fill!" + (("added") + ("date" . "2026-08-01") + ("modules_added") + ("moved") + ("notes" + . + "Removed the ancient tracked vendor/jsqlite snapshot. Builds now fetch the real jerboa-sqlite repository from Forgejo at an immutable commit and tree recorded in vendor-lock.env; vendor and security gates verify both identities and reject dirty checkouts.") + ("removed") + ("renamed") + ("tier_changes") + ("tools_added" "make vendor-deps" "make vendor-clean") + ("version" . #f)) + (("added" ("bytevector-u8-set!" "bytevector-fill!" "bytevector-copy!" "BytesBuilder" "make-bytes-builder" "bytes-builder-push!" "bytes-builder-extend!" "bytes-builder->bytes" "StringBuilder" --- a/docs/reviews/2026-07-27-vendor-jsqlite-tcb.md +++ b/docs/reviews/2026-07-27-vendor-jsqlite-tcb.md @@ -5,8 +5,13 @@ Review max age: 90 days SQLite compatibility target: 3.54.0 Decision: accepted-risk Runtime dependency: none on libsqlite3 +Source repository: https://git.jerboa.sh/ober/jerboa-sqlite +Pinned commit: 589d8ad57edcdd0aac4e9654f7b2e48f0d577a87 +Pinned tree: d5d28877fc965c5005003024d2e02b65d1a88905 -`vendor/jsqlite` remains in the Jerboa TCB for now. It is a pure Jerboa +`vendor/jsqlite` remains in the Jerboa TCB, but its source is no longer stored +in this repository. `make vendor-deps` fetches and verifies the immutable +commit and tree above from the real `jerboa-sqlite` Forgejo repository. It is a pure Jerboa SQLite-compatible engine, not a dynamically linked copy of upstream `libsqlite3`, so `cargo audit` and system SQLite package scanners do not cover it. The release gate therefore treats it as an internal database engine with a @@ -35,5 +40,6 @@ Current review notes: Release rule: `make audit` runs `support/check-vendor-jsqlite-security.sh`. The check fails -when this decision record is missing, the README compatibility target drifts, +when the fetched commit/tree differs from `vendor-lock.env`, the checkout is +dirty, this decision record is missing, the README compatibility target drifts, or the advisory review is older than `Review max age`. --- a/support/check-vendor-jsqlite-security.sh +++ b/support/check-vendor-jsqlite-security.sh @@ -3,6 +3,25 @@ set -eu doc="${1:-docs/reviews/2026-07-27-vendor-jsqlite-tcb.md}" readme="${2:-vendor/jsqlite/README.md}" +lock="${3:-vendor-lock.env}" + +# shellcheck disable=SC1090 +. "$lock" + +actual_commit=$(git -C "$(dirname "$readme")" rev-parse HEAD) +actual_tree=$(git -C "$(dirname "$readme")" rev-parse 'HEAD^{tree}') +[ "$actual_commit" = "$JSQLITE_COMMIT" ] || { + echo "vendor-jsqlite-security: checkout commit $actual_commit does not match lock $JSQLITE_COMMIT" >&2 + exit 1 +} +[ "$actual_tree" = "$JSQLITE_TREE" ] || { + echo "vendor-jsqlite-security: checkout tree $actual_tree does not match lock $JSQLITE_TREE" >&2 + exit 1 +} +git -C "$(dirname "$readme")" diff --quiet HEAD -- || { + echo "vendor-jsqlite-security: fetched checkout has local modifications" >&2 + exit 1 +} python3 - "$doc" "$readme" <<'PY' import datetime as dt new file mode 100755 --- /dev/null +++ b/support/fetch-vendor.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck disable=SC1091 +. "$repo_root/vendor-lock.env" + +component=${1:-all} +vendor_root=$repo_root/vendor +mkdir -p "$vendor_root" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +clone_locked() { + name=$1 + repository=$2 + commit=$3 + expected_tree=$4 + destination=$5 + + if [ -d "$destination/.git" ]; then + actual_commit=$(git -C "$destination" rev-parse HEAD) + actual_tree=$(git -C "$destination" rev-parse 'HEAD^{tree}') + [ "$actual_commit" = "$commit" ] || fail "$name checkout is $actual_commit, expected $commit; run make vendor-clean" + [ "$actual_tree" = "$expected_tree" ] || fail "$name tree is $actual_tree, expected $expected_tree" + git -C "$destination" diff --quiet HEAD -- || fail "$name checkout has local modifications" + return + fi + [ ! -e "$destination" ] || fail "$destination exists without verifiable Git metadata; run make vendor-clean" + + tmp=$vendor_root/.fetch-$name-$$ + trap 'rm -rf "$tmp"' EXIT HUP INT TERM + git init -q "$tmp" + git -C "$tmp" remote add origin "$repository" + git -C "$tmp" fetch -q --depth 1 --filter=blob:none origin "$commit" + git -C "$tmp" checkout -q --detach FETCH_HEAD + actual_commit=$(git -C "$tmp" rev-parse HEAD) + actual_tree=$(git -C "$tmp" rev-parse 'HEAD^{tree}') + [ "$actual_commit" = "$commit" ] || fail "$name fetched unexpected commit $actual_commit" + [ "$actual_tree" = "$expected_tree" ] || fail "$name fetched unexpected tree $actual_tree" + mv "$tmp" "$destination" + trap - EXIT HUP INT TERM +} + +case "$component" in + all|jsqlite) + clone_locked jsqlite "$JSQLITE_REPOSITORY" "$JSQLITE_COMMIT" "$JSQLITE_TREE" "$vendor_root/jsqlite" + ;; + *) fail "unknown vendor component: $component" ;; +esac new file mode 100644 --- /dev/null +++ b/vendor-lock.env @@ -0,0 +1,4 @@ +# Immutable Forgejo source input used by Makefile vendor targets. +JSQLITE_REPOSITORY=https://git.jerboa.sh/ober/jerboa-sqlite.git +JSQLITE_COMMIT=589d8ad57edcdd0aac4e9654f7b2e48f0d577a87 +JSQLITE_TREE=d5d28877fc965c5005003024d2e02b65d1a88905 deleted file mode 100644 --- a/vendor/jsqlite/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# Transpiled / compiled artifacts -/lib/ -*.sls -*.so -*.dylib -*.wpo -**/.jerbuild-hashes - -# OS / editor -.DS_Store -*~ - -# Local scratch logs (never committed) -/.scratch/ - -# Database sidecar files created at runtime (locking / WAL / rollback journal) -*-lock -*-wal -*-shm -*-journal deleted file mode 100644 --- a/vendor/jsqlite/Makefile +++ /dev/null @@ -1,139 +0,0 @@ -# jerboa-sqlite -- a SQLite3-compatible database engine in Jerboa Scheme. -# -# jerbuild bundles Chez Scheme + the jerboa stdlib, so building jsqlite needs -# only `jerbuild` on PATH. The differential ("diff") test lane additionally -# needs the jerboa-sqlite reference (a thin FFI over the system libsqlite3), -# checked out as a sibling directory; it is TEST-ONLY (see the working rule in -# OPUS_4_8_SQLITE_HANDOFF.md: never call libsqlite3 from production modules). - -JERBUILD ?= jerbuild -JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) -ifeq ($(JH),) -$(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) -endif - -SRC_DIR := $(CURDIR)/src -TOOLS_DIR := $(CURDIR)/tools -REF_DIR := $(CURDIR)/../jerboa-sqlite-ffi -REF_LIB := $(REF_DIR)/lib - -# Production execution: just our source + the jerboa stdlib. -LIBDIRS := $(SRC_DIR):$(JH)/lib -# Differential execution: also the test tools + the reference oracle library. -DIFF_LIBDIRS := $(SRC_DIR):$(TOOLS_DIR):$(REF_LIB):$(JH)/lib - -UNIT_TESTS := $(wildcard tests/unit/*.ss) -DIFF_TESTS := $(wildcard tests/diff/*.ss) - -.PHONY: all build unit diff test oracle fuzz corrupt dmlfuzz concurrency cshim bench robustness clean help -.DEFAULT_GOAL := help - -all: test - -# Compile-check: import every production module (R6RS libraries in .ss files -# are consumed directly via libdirs, so there is no transpile step). -build: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/build-check.ss - -# Unit tests run directly against src/ (jerbuild exec compiles on the fly). -unit: - @fail=0; for t in $(UNIT_TESTS); do \ - echo "== unit: $$t =="; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" $$t || fail=1; \ - done; \ - if [ $$fail -ne 0 ]; then echo "UNIT FAILURES"; exit 1; fi - -# Build the reference oracle (shim + transpiled lib) for differential tests. -oracle: - @if [ ! -d "$(REF_DIR)" ]; then \ - echo "reference $(REF_DIR) not found"; exit 1; fi - $(MAKE) -C $(REF_DIR) build - -# Differential tests compare jsqlite against the reference engine. They are -# skipped (not failed) when the reference checkout is unavailable. -diff: - @if [ ! -d "$(REF_DIR)" ]; then \ - echo "SKIP diff: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP diff: reference build failed"; exit 0; }; \ - fail=0; for t in $(DIFF_TESTS); do \ - echo "== diff: $$t =="; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" $$t || fail=1; \ - done; \ - if [ $$fail -ne 0 ]; then echo "DIFF FAILURES"; exit 1; fi - -test: unit diff - -# Robustness tools (not part of the default gate; longer-running, explicit). -# Override count/seed with `make fuzz N=5000 SEED=7`. -fuzz: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP fuzz: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP fuzz: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/fuzz.ss $(N) $(SEED) - -# Malformed-image corpus test; needs only jsqlite (no oracle). -corrupt: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/corrupt-test.ss $(N) $(SEED) - -# Multi-process write-locking stress test: 4 processes hammer one file, then -# verify no writes were lost. Needs only jsqlite (no oracle). `make concurrency N=40`. -concurrency: - @P=/tmp/jsqlite-concurrency.db; n=$${N:-40}; \ - for mode in "" wal; do \ - echo "== concurrency (journal mode: $${mode:-rollback}) =="; \ - rm -f $$P $$P-lock $$P-journal $$P-wal $$P-shm; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss init $$P; \ - for id in 1 2 3 4; do \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss worker $$id $$n $$P $$mode >/dev/null & \ - done; wait; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss verify $$P 4 $$n || exit 1; \ - done - -# Write-path / B-tree property test vs the oracle (`make dmlfuzz N=2000 SEED=7`). -dmlfuzz: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP dmlfuzz: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP dmlfuzz: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/dml-fuzz.ss $(N) $(SEED) - -# C ABI shim: compile the C driver (shim/driver.c) and have it drive jsqlite -# prepared statements through the jsq_api function-pointer table. Needs a C -# compiler (cc); no oracle. -cshim: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/cshim-test.ss - -# Performance benchmark vs upstream SQLite, using SQLite's own 16-step -# "Database Speed Comparison" workload (docs/benchmarks.md). N is the -# big-transaction insert count (default 1000); `make bench N=2000` for more. -bench: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP bench: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP bench: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/bench.ss $(N) - -robustness: corrupt dmlfuzz concurrency fuzz - -clean: - rm -rf lib - find . \( -name '*.wpo' -o -name '*.so' \) -delete 2>/dev/null || true - -help: - @echo "jsqlite -- SQLite3-compatible engine in Jerboa Scheme" - @echo "" - @echo " make build Compile-check all production modules" - @echo " make unit Run unit tests (tests/unit/*.ss)" - @echo " make diff Run differential tests vs the jerboa-sqlite oracle" - @echo " make test Run unit + diff (default acceptance gate)" - @echo " make fuzz Differential SQL fuzz vs the oracle (N=, SEED=)" - @echo " make corrupt Malformed-database corpus robustness test (N=, SEED=)" - @echo " make dmlfuzz Write-path / B-tree property test vs the oracle (N=, SEED=)" - @echo " make concurrency Multi-process write-locking stress test (N=)" - @echo " make cshim C ABI shim: a C driver runs jsqlite prepared statements (needs cc)" - @echo " make bench Performance benchmark vs upstream SQLite (N=)" - @echo " make robustness corrupt + dmlfuzz + fuzz" - @echo " make clean Remove build artifacts" deleted file mode 100644 --- a/vendor/jsqlite/README.md +++ /dev/null @@ -1,123 +0,0 @@ -# jerboa-sqlite - -A SQLite3-compatible database engine implemented from scratch in **Jerboa -Scheme** — not a binding over `libsqlite3`, but a reimplementation of the -engine: tokenizer, parser, value/type system, VDBE-style virtual machine, -B-tree storage, pager, and the `sqlite3_*`-shaped API. - -Pinned compatibility target: **SQLite 3.54.0** (the upstream checkout in -`../sqlite`). The thin FFI wrapper in `../jerboa-sqlite-ffi` is used **only** as a -differential-test oracle, never as a runtime dependency. - -## Status - -Built in phases (see `OPUS_4_8_SQLITE_HANDOFF.md` for the full ladder). - -| Phase | Scope | State | -|------|-------|-------| -| 0 | Scaffold, value/type foundations, oracle harness | ✅ done | -| 1 | Values, tokenizer, parser, in-memory evaluator | ✅ done | -| 2 | SQLite file reader (header, records, B-tree, schema) | ✅ done | -| 3 | Read-only SELECT over tables (WHERE/ORDER/LIMIT/DISTINCT/joins) | ✅ done | -| 4 | Write path: CREATE TABLE/CTAS / CREATE/DROP INDEX / INSERT / UPDATE / DELETE, NOT NULL, freelist basics (experimental) | ✅ done | -| 7a | Aggregates + GROUP BY/HAVING; BETWEEN/IN/LIKE/GLOB/COLLATE + scalar functions; compound UNION/INTERSECT/EXCEPT; derived tables, non-recursive CTEs, `WITH` on INSERT SELECT, and views | ✅ done | -| next | Remaining foreign-key edge cases, index planner, locking/fsync/WAL hardening | planned | - -What works today: the full value/type system (NULL/INTEGER/REAL/TEXT/BLOB with -SQLite truthiness, numeric coercion, storage-class comparison ordering and -three-valued logic); a complete SQLite tokenizer (SQLite keywords, all literal and -operator forms); a precedence-correct expression parser; and an evaluator -covering arithmetic (with integer-overflow→real and `/0`→NULL), bitwise ops, -`CAST` affinity, SQLite comparison affinity for direct table columns and rowid, -`CASE`, `COLLATE BINARY/NOCASE/RTRIM` including declared column defaults for -direct table scans, bind parameters, and a core set of scalar functions. -The prepared-statement API covers prepare/step/reset/finalize, generic and typed -bind helpers, column accessors, `sqlite-exec`/`sqlite-query`, change counters, -read-only/open-v2 modes, and connection error accessors. It runs table-less `SELECT`s in memory, verified -expression-by-expression against the SQLite 3.54.0 reference in the differential -test lane. It also **reads real SQLite database files** — parsing the header, -records, and table B-trees (including interior pages and overflow chains) — so -`SELECT <cols/*> FROM <table>` returns rows from files written by upstream -SQLite, byte-for-byte verified against the reference. Top-level, compound, -derived, scalar, and CTE `VALUES` queries are supported. The write path creates -rowid tables (including `AUTOINCREMENT`/`sqlite_sequence`), materializes -`CREATE TABLE ... AS SELECT`/`VALUES`, supports `STRICT` rowid tables and -STORED/VIRTUAL generated columns, stores/query schema-only views, supports -`ALTER TABLE ... RENAME TO ...`, `ALTER TABLE ... ADD COLUMN`, and -`ALTER TABLE ... RENAME COLUMN ... TO ...`, plus added-column CHECK validation -and added-column REFERENCES/default validation, safe `DROP COLUMN` cases, and creates user index -roots (`CREATE INDEX`, `CREATE UNIQUE INDEX`, `DROP INDEX`) while maintaining -those indexes across INSERT/UPDATE/DELETE for files that pass upstream -`PRAGMA integrity_check`, including `INSERT ... SELECT`, `WITH`-fed INSERT SELECT, -`UPDATE ... FROM`, simple UPSERT, DML `RETURNING`, and multi-page user -indexes; low-level table rowid lookups and index B-tree scans handle upstream -overflow payloads, while multi-level and overflow index writes pass upstream -integrity checks. The planner can use simple rowid equality point lookups, -mandatory `INDEXED BY`, simple single-table constrained index scans, and -covered-projection index scans; broader cost-based planning is still future work. -Foreign-key checks are -available when `PRAGMA foreign_keys=ON` for ordinary rowid tables, including -parent-key mismatch validation and immediate `ON DELETE`/`ON UPDATE` -`RESTRICT`/`CASCADE`/`SET NULL`/`SET DEFAULT` actions, including `DROP TABLE`'s -implicit delete behavior; -`DEFERRABLE INITIALLY DEFERRED` constraints are checked at COMMIT or outermost -savepoint RELEASE, and `PRAGMA defer_foreign_keys` can defer otherwise-immediate -checks for the current transaction. File commits use a simple DELETE-mode -rollback journal and recover a leftover hot journal on open; full locking/fsync -and WAL behavior are still future work. - -## Layout - -``` -src/jsqlite/ production engine modules (never touch libsqlite3) - constants.ss result codes, open flags, datatype tags (SQLite 3.54.0) - error.ss error condition type + result-code plumbing - value.ss values, storage classes, truthiness, coercions - keywords.ss SQL keyword table - tokenize.ss hand-coded tokenizer (mirrors src/tokenize.c) - ast.ss parse.ss AST records + precedence-climbing parser - eval.ss expression evaluator (SQLite semantics) - record.ss page.ss varint/record codec; file header + page geometry - btree.ss table/index B-tree read cursors (interior/leaf, overflow, rowid lookup) - schema.ss sqlite_schema reader + CREATE TABLE column resolution - exec.ss SELECT executor (joins, WHERE, ORDER BY, LIMIT, DISTINCT) - writer.ss mutable DB image: header, page alloc, freelist, table/index rebuilds - dml.ss CREATE TABLE/INDEX / INSERT / UPDATE / DELETE execution - api.ss open/prepare/step/column/finalize (sqlite3_*-shaped) -tools/ test-only tooling - oracle-runner.ss differential oracle: jsqlite vs jerboa-sqlite reference -tests/ - unit/ per-module unit tests - diff/ differential tests against the reference engine - fileformat/ file round-trip tests (later phases) - upstream/ adapted upstream Tcl-test cases (later phases) -docs/ - architecture.md module map + SQLite pipeline - test-plan.md the five test lanes + normalization rules - compatibility-matrix.md feature/status tracking -``` - -## Build & test - -Requires [`jerbuild`](https://git.sr.ht/~lisp) on `PATH`. The differential lane -additionally requires the `../jerboa-sqlite-ffi` reference checkout (a C compiler + -system `libsqlite3`); it is skipped if that checkout is absent. - -```sh -make unit # unit tests only (no libsqlite3 needed) -make diff # differential tests vs the reference oracle -make test # unit + diff (acceptance gate) -make build # compile-check all production modules -``` - -## Design notes - -- **Values** are native Scheme types plus a single `sql-null` sentinel: - INTEGER = exact integer, REAL = flonum, TEXT = string, BLOB = bytevector. - SQLite has no boolean type; TRUE/FALSE are integers 1/0, and NULL drives - three-valued logic. -- **Compatibility is tracked on three surfaces** — SQL behavior, on-disk file - format, and API shape — each with its own tests. See `docs/test-plan.md`. -- The engine mirrors SQLite's architecture (SQL → tokenize → parse → codegen → - VDBE → B-tree → pager) so that hard compatibility cases (affinity, the file - format, transactions) have a place to live correctly. deleted file mode 100644 --- a/vendor/jsqlite/docs/architecture.md +++ /dev/null @@ -1,68 +0,0 @@ -# Architecture - -jsqlite mirrors SQLite's own architecture rather than interpreting SQL directly, -because full compatibility (affinity, the file format, transactions, triggers) -is far easier across a prepared-statement / virtual-machine boundary. - -## Pipeline - -``` -SQL text - -> tokenizer (tokenize.ss) bytes -> tokens - -> parser (parse.ss) tokens -> AST (ast.ss) - -> code generator (codegen.ss) AST -> VDBE program - -> VDBE (vdbe.ss) register machine, statement lifecycle - |-- expressions (expr.ss), functions (functions.ss) - |-- cursors over... - -> B-tree (btree.ss) table/index cursors + mutations - -> pager (pager.ss) page cache, rollback journal, locks - -> file/VFS (page.ss) header + page encode/decode -``` - -The public API (`api.ss`) is the prepared-statement boundary: `open -> prepare --> step -> column-* -> reset/finalize`, shaped like `sqlite3_*`. - -## Module map (target) - -| Module | Responsibility | State | -|--------|----------------|-------| -| `constants.ss` | result codes, open flags, datatype tags | ✅ | -| `error.ss` | error condition type, primary/extended codes | ✅ | -| `value.ss` | storage classes, truthiness, numeric coercion | ✅ (core) | -| `api.ss` | database/statement lifecycle, transactions, utility statements | ✅ (bootstrap) | -| `encoding.ss` | UTF-8 (UTF-16 later) | planned | -| `collation.ss` | BINARY / NOCASE / RTRIM | implemented in `eval.ss`/schema metadata; standalone module/custom collations later | -| `tokenize.ss` | SQLite tokenizer | Phase 1 | -| `parse.ss` / `ast.ss` | parser + AST | Phase 1 | -| `expr.ss` | expression analysis / eval | Phase 1 | -| `vdbe.ss` / `codegen.ss` | VM + code generation | Phase 1 | -| `record.ss` | record varints + serial types | Phase 2 | -| `page.ss` / `btree.ss` | file format + table/index B-tree scans and rowid lookup | Phase 2/4 | -| `pager.ss` / `wal.ss` | pager, journal, locks, WAL | Phase 4/6/9 | -| `schema.ss` | sqlite_schema cache, column/constraint metadata | Phase 2/3/5 | -| `planner.ss` | WHERE / index selection | Phase 3/5 | -| `functions.ss` | scalar/aggregate/window functions | Phase 1/7 | -| `pragma.ss` | PRAGMA handlers | Phase 8 (table/index introspection, cookies, journal/foreign-key toggles) | - -## Value representation - -A SQLite value is one of five storage classes, represented with native Scheme -types plus a single NULL sentinel (`value.ss`): - -| Storage class | Jerboa representation | -|---------------|-----------------------| -| NULL | the unique `sql-null` object | -| INTEGER | exact integer (64-bit signed range enforced in arithmetic) | -| REAL | flonum (IEEE-754 binary64) | -| TEXT | string | -| BLOB | bytevector | - -SQLite has no boolean type. `sql-truthy?` returns `#t`/`#f`/`'null` to model -three-valued logic. TEXT/BLOB are coerced to numbers by their longest leading -numeric run (verified ground truth: `'3abc'` is true, `'0abc'`/`'abc'` false; -a blob's bytes are read as text, so `x'31'` = `"1"` is true). - -## Working rule - -`libsqlite3` is reached **only** from `tools/` and `tests/` (the differential -oracle). Production modules under `src/jsqlite/` never link or call it. deleted file mode 100644 --- a/vendor/jsqlite/docs/compatibility-matrix.md +++ /dev/null @@ -1,135 +0,0 @@ -# Compatibility matrix - -Pinned to **SQLite 3.54.0**. Tracks feature status across the three compliance -surfaces. Columns: feature, status, oracle/diff test, upstream tests, notes. - -Status legend: ✅ done · 🟡 partial · ⬜ planned. - -## Values & typing - -| Feature | Status | Diff/unit test | Upstream tests | Notes | -|---------|--------|----------------|----------------|-------| -| Storage classes (NULL/INT/REAL/TEXT/BLOB) | ✅ | `unit/test-value` | `types*.test` | native repr + `sql-null` sentinel | -| `typeof` names | ✅ | `unit/test-value`, `diff/test-expr` | `e_expr.test` | | -| Truthiness (3-valued) | ✅ | `unit/test-value` | `e_expr.test` | TEXT/BLOB via numeric prefix | -| Numeric coercion (prefix) | ✅ | `unit/test-value` | `cast.test` | sqlite3AtoF/Atoi64 semantics | -| Numeric affinity | 🟡 | `unit/test-value`, `diff/test-select` | `affinity*.test` | insert/CAST and direct-table comparison affinity covered; broader upstream affinity corpus later | -| Comparison ordering & 3-valued logic | ✅ | `diff/test-expr`, `diff/test-select` | `e_expr.test` | NULL<number<text<blob; column-affinity conversions for `=`,`IS`,`BETWEEN`,`IN` lists, and base `CASE`; `IS TRUE/FALSE`,`AND`/`OR` | -| Arithmetic (+ - * / %), overflow→real, /0→NULL | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | | -| Bitwise (& \| << >> ~) | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | 64-bit wrap, neg-shift reverses | -| `CAST` (INT/REAL/TEXT/BLOB/NUMERIC affinity) | ✅ | `unit/test-eval`, `diff/test-expr` | `cast.test` | NUMERIC int-reduction for text/blob | -| Collations (BINARY/NOCASE/RTRIM) | ✅ | `unit/test-eval`, `unit/test-fileformat`, `diff/test-expr`, `diff/test-select`, `diff/test-constraints`, `diff/test-subquery`, `diff/test-compound` | `collate*.test` | explicit `COLLATE`; declared-column defaults; derived/compound propagation | - -## SQL surface - -| Feature | Status | Diff/unit test | Upstream tests | Notes | -|---------|--------|----------------|----------------|-------| -| Tokenizer | ✅ | `unit/test-tokenize` | tokenize-derived | full token set, SQLite keywords, hex/blob/vars | -| Parser / AST (expressions) | ✅ | `unit/test-parse` | — | Pratt parser, SQLite precedence | -| Table-less `SELECT <exprs>` / `VALUES(...)` | ✅ | `diff/test-expr`, `diff/test-select-constants` | `select1.test` | SELECT column names = source spans; VALUES column names = `columnN`; scalar/IN `VALUES` subqueries | -| Bind parameters (`?`,`?N`,`:x`,`@x`,`$x`) | ✅ | `unit/test-api`, `unit/test-parse` | `bind*.test` | SQLite numbering rules | -| `CASE` expression | ✅ | `unit/test-eval`, `diff/test-expr` | `e_expr.test` | base + searched forms | -| `BETWEEN` / `IN (list/table)` / `LIKE` / `GLOB` (+ NOT, ESCAPE) | ✅ | `diff/test-functions`, `diff/test-subquery`, `unit/test-pragma` | `like*.test`, `in*.test` | LIKE is ASCII case-insensitive by default and honors connection-local `PRAGMA case_sensitive_like`; GLOB classes; IN NULL semantics and direct-table comparison affinity; `x IN table` shorthand | -| `COLLATE` (BINARY/NOCASE/RTRIM) | ✅ | `unit/test-parse`, `unit/test-eval`, `diff/test-expr`, `diff/test-select` | `collate*.test` | comparisons, `IS` / `IS DISTINCT FROM`, `BETWEEN`, `IN` lists, ORDER/GROUP/DISTINCT, min/max | -| Scalar functions | 🟡 | `diff/test-functions`, `diff/test-date`, `unit/test-functions` | `func*.test`, `date*.test`, `timediff*.test` | typeof/abs/length/octet_length/lower/upper/coalesce/ifnull/nullif/iif/if/likely/unlikely/likelihood/concat/concat_ws/substr/substring/replace/instr/trim/ltrim/rtrim/min/max/round/sign/random/randomblob/hex/unhex/quote/unistr_quote/zeroblob/printf/format/char/unicode/unistr/sqlite_version/sqlite_source_id/changes/total_changes/last_insert_rowid/date/time/datetime/julianday/unixepoch/strftime/timediff/current_date/current_time/current_timestamp/like/glob plus SQLite math functions (`acos`, `acosh`, `asin`, `asinh`, `atan`, `atan2`, `atanh`, `ceil`, `ceiling`, `cos`, `cosh`, `degrees`, `exp`, `floor`, `ln`, `log`, `log10`, `log2`, `mod`, `pi`, `pow`, `power`, `radians`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `trunc`); REAL-to-text covers SQLite-style 15-significant-digit fixed/exponential cases; localtime/utc modifiers covered against host SQLite; broader upstream corpus later | -| `SELECT <cols/*/t.*> FROM` tables | ✅ | `diff/test-select`, `unit/test-exec` | `select*.test` | full scan; projection + expressions; aliases and parenthesized single table refs | -| `INDEXED BY` / `NOT INDEXED` table qualifiers | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-select` | `indexedby*.test` | SELECT/UPDATE-FROM table refs plus UPDATE/DELETE targets parse and validate that `INDEXED BY` names an index on the base table; SELECT table scans honor a mandatory `INDEXED BY` scan order; partial-index hints require a conservative exact-conjunct predicate proof or raise `no query solution`; simple unhinted single-table SELECTs can scan a usable user index for leading term constraints or covered projections, including expression and partial constrained searches, with `NOT INDEXED` opting out; broader cost-based selection later | -| `EXPLAIN QUERY PLAN` | 🟡 | `unit/test-parse`, `diff/test-select` | `eqp*.test` | SELECT/VALUES/WITH query wrapper returns SQLite-shaped `id,parent,notused,detail` rows for constant rows, VALUES, base-table scans, rowid primary-key searches, forced index scans/searches, covering index scans/searches, simple unhinted user-index searches, and leading composite-index constraint details; full VDBE `EXPLAIN` and complete multi-node planner trees later | -| `WHERE` filter | ✅ | `diff/test-select`, `unit/test-exec`, `unit/test-api` | `where*.test` | three-valued truthiness; direct-table comparison affinity; source-first SELECT-list alias fallback; simple single-table integer rowid/IPK equality predicates use table B-tree point lookup | -| `ORDER BY` (multi-key, ASC/DESC, ordinal, alias) | ✅ | `diff/test-select`, `unit/test-exec` | — | stable; integer and signed-integer ordinals; output aliases before source columns for bare unqualified names, source-first alias fallback inside expressions; default NULL ordering plus explicit `NULLS FIRST`/`NULLS LAST`; explicit COLLATE | -| `LIMIT` / `OFFSET` | ✅ | `diff/test-select` | — | `LIMIT a,b`; negative = unlimited | -| `DISTINCT` | ✅ | `diff/test-select` | — | numeric equality; NULLs equal; explicit COLLATE | -| Inner / cross joins + table aliases + qualified cols | ✅ | `diff/test-select` | — | nested loop; ON folded into filter; source-first SELECT-list alias fallback in ON/filter expressions | -| LEFT / RIGHT / FULL OUTER joins; `USING` / NATURAL joins | 🟡 | `diff/test-select`, `unit/test-parse` | `join*.test` | ON predicates preserve unmatched rows and can use source-first output aliases; `USING`/NATURAL hide duplicate right columns and coalesce visible output; broader upstream join corpus later | -| Aggregates (count/sum/total/avg/min/max/group_concat/string_agg, DISTINCT) | ✅ | `diff/test-aggregate`, `unit/test-aggregate`, `diff/test-functions` | `func*.test` | count(*)/count(); empty/NULL semantics; `FILTER (WHERE ...)`; aggregate-call `ORDER BY`; per-row `group_concat` separators; `string_agg(X,Y)` is the SQL-standard spelling of `group_concat`; a DISTINCT aggregate with more than one argument is rejected like SQLite | -| JSON scalar functions | 🟡 | `diff/test-json` | `json*.test` | `json` (validate + minify, preserving source string escapes like SQLite), `json_valid`, `json_type` (root or path), `json_extract` (single path returns the SQL value, multiple paths return a JSON array, `$`/`.key`/`["key"]`/`[i]`/`[#-i]` paths), `json_array_length`, `json_quote`, `json_array`, `json_object`, `json_set`/`json_insert`/`json_replace` (functional edits, `[#]` appends, out-of-range numeric index is a no-op), `json_remove`, and `json_patch` (RFC 7396 merge), plus the `->` (JSON-valued) and `->>` (SQL-valued) operators whose right operand is an integer index, a `$`-path, or a bare label; the `json_group_array` and `json_group_object` aggregates (with `FILTER`/`ORDER BY`/`DISTINCT`, `json_group_array` keeping NULLs), and the `json_each`/`json_tree` table-valued functions (see the table-valued-functions row); an RFC-8259 parser/minifier backs them and the oracle verifies output byte-for-byte. A value returned by a JSON function carries SQLite's JSON subtype (tracked per statement by result-string identity), so `json_array(json_object(...))`, `json_set(x,p,json(...))`, and `json_extract` of a container embed as JSON rather than double-encoding, while plain strings stay quoted; `subtype(X)` reports `74` for a JSON-subtyped text value and `0` otherwise. The parsing functions accept **JSON5** input (comments, unquoted/single-quoted keys and strings, trailing commas, hex and `.5`/`5.` numbers, `+`/`Infinity`/`NaN`), output stays canonical, and `json_valid` honors the strict (default) vs JSON5 (`0x02`) flag bit. The **JSONB** binary format is supported: a byte-for-byte-compatible encoder/decoder backs `jsonb`/`jsonb_extract`/`jsonb_array`/`jsonb_object`/`jsonb_set`/`jsonb_insert`/`jsonb_replace`/`jsonb_remove`/`jsonb_patch` (returning JSONB blobs with the JSON subtype), every `json_*` function reads a JSONB blob argument, the `jsonb_group_array`/`jsonb_group_object` aggregates produce JSONB, and `hex(jsonb(...))` matches upstream exactly | -| `GROUP BY` / `HAVING` | ✅ | `diff/test-aggregate`, `diff/test-select`, `unit/test-aggregate` | — | aggregates inside expressions; GROUP BY ordinals and source-first alias fallback; HAVING output aliases; explicit COLLATE grouping | -| Compound `UNION`/`UNION ALL`/`INTERSECT`/`EXCEPT` | ✅ | `diff/test-compound`, `unit/test-compound` | `select4*.test` | trailing ORDER BY/LIMIT with ordinal, alias, and output-expression resolution; numeric dedupe; SELECT and VALUES arms | -| Subqueries: scalar, `EXISTS`, `IN (SELECT)` (+ correlation) | ✅ | `diff/test-subquery`, `unit/test-subquery` | `subquery*.test` | lexically-scoped column resolution; `IN (SELECT)` uses output affinity for single-column comparisons | -| Subqueries in FROM (derived tables) | ✅ | `diff/test-subquery`, `unit/test-subquery` | `subquery*.test` | SELECT/compound/VALUES derived tables preserve output collation and affinity; no synthetic rowid | -| `WITH` common table expressions | 🟡 | `unit/test-parse`, `unit/test-subquery`, `unit/test-write`, `diff/test-select`, `diff/test-write`, `diff/test-returning`, `diff/test-select-constants` | `with*.test` | SELECT/VALUES CTEs, including column aliases, compound and nested WITH CTE bodies, derived-table WITH queries, and `WITH RECURSIVE` anchors plus one or more recursive arms under `UNION`/`UNION ALL`, with queue `ORDER BY`/`LIMIT`/`OFFSET`; `WITH` on INSERT, UPDATE, and DELETE; broader recursive edge cases later | -| Window functions | 🟡 | `unit/test-parse`, `diff/test-functions` | `window*.test` | inline or named `OVER` clauses with default frames for built-ins (`row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`, `lead`, `lag`, `first_value`, `last_value`, `nth_value`) and aggregate windows (`count`, `sum`, `total`, `avg`, `min`, `max`, `group_concat`) including aggregate `FILTER (WHERE ...)`; named base-window inheritance for `WINDOW` clauses and inline `OVER (base ...)`; explicit `ROWS`, `GROUPS`, and single-term `RANGE` frame clauses with `EXCLUDE` variants for aggregate and value windows; window functions over grouped aggregate result rows; `FILTER` on non-aggregate built-ins and `DISTINCT` window aggregates are rejected like SQLite; broader upstream window corpus later | -| `CREATE VIEW` / `DROP VIEW`; selecting from views | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-select`, `diff/test-write` | `view*.test` | schema-only views with optional output column list; SELECT/WITH/VALUES view bodies; circular view reads are rejected; INSTEAD OF triggers can route view INSERT/UPDATE/DELETE; `CREATE TEMP VIEW` lives in the temp schema (listed in `sqlite_temp_master`, never persisted) and its body may read temp or main tables | -| `CREATE TABLE` / `CREATE TABLE ... AS SELECT` (+ IF NOT EXISTS) | 🟡 | `diff/test-write`, `diff/test-strict`, `unit/test-fileformat`, `unit/test-write` | `createtab*.test`, `strict*.test` | rowid tables; built-in `main.` qualifier; duplicate column names rejected; AUTOINCREMENT/sqlite_sequence; CTAS materializes SELECT/WITH/VALUES output and generated column names; declared collations captured; `STRICT` rowid tables enforce storage classes and preserve `ANY`; STORED and VIRTUAL generated columns are dependency-ordered on INSERT/UPDATE, virtual columns are omitted from physical records, generated-column indexes/UNIQUE constraints are maintained, and upstream readback/integrity_check passes; `WITHOUT ROWID` is supported (see its own row); TEMP/virtual tables are explicit unsupported cases | -| WITHOUT ROWID tables | 🟡 | `unit/test-without-rowid`, `diff/test-without-rowid` | `withoutrowid*.test` | stored as a PRIMARY-KEY-keyed index B-tree (PK columns first, then the remaining columns); single/composite PKs with per-column `ASC`/`DESC` and `COLLATE`; PK columns are implicitly NOT NULL; duplicate-PK raises `SQLITE_CONSTRAINT_PRIMARYKEY`/`_UNIQUE`; `rowid`/`oid`/`_rowid_` are not columns; secondary and UNIQUE indexes are keyed by the PK (not a rowid); INSERT/SELECT/UPDATE/DELETE plus upstream `integrity_check` and bidirectional readback round-trip pass; missing PRIMARY KEY and AUTOINCREMENT are rejected like upstream; the rowid index planner does not apply, so reads full-scan | -| TEMP tables / indexes / views (in-memory `temp` schema) | 🟡 | `unit/test-temp`, `unit/test-parse` | `temp*.test` | `CREATE TEMP TABLE/INDEX/VIEW` live in a separate in-memory image that shadows `main` for unqualified names and never persists to disk; full INSERT/SELECT/UPDATE/DELETE/DROP, secondary indexes on temp tables, and same-name shadowing; a plain `CREATE INDEX` lands wherever its target table lives, and `CREATE TABLE temp.x` / `DROP TABLE temp.x` route to the temp schema; unqualified `sqlite_master` reports `main` while `sqlite_temp_master` / `sqlite_temp_schema` report the temp schema (empty until a temp object exists, and not writable); `PRAGMA database_list` reports `temp` once a temp object exists; `temp.`-qualified table and 3-part column references resolve to the temp schema, a single statement may join temp and main tables, and BEGIN/COMMIT/ROLLBACK/SAVEPOINT cover temp and main together (one snapshot spans both); a 3-part column ref (`main.both.b` vs `temp.both.b`) disambiguates two same-named tables joined from different schemas, while the bare `both.b` still reaches its table and is reported ambiguous when both schemas are joined | -| ATTACH / DETACH | 🟡 | `unit/test-attach`, `diff/test-attach`, `diff/test-crossdb` | `attach*.test` | `ATTACH '<file>'` or `':memory:' AS <name>` adds a named database (searched after temp/main for unqualified names) and `DETACH` removes it; `CREATE TABLE <db>.t` directs a new table into a database; file-backed attachments persist on write and pass upstream `integrity_check`/readback; `PRAGMA database_list` reports them at seq 2+. Reads resolve each table reference to its database via a per-query catalog, so a single SELECT/join/subquery can read across main, temp, and attached databases (verified against upstream in `diff/test-crossdb`). Each write (INSERT/UPDATE/DELETE/DDL) targets one database, but its source may read others: `INSERT INTO aux.t SELECT … FROM main.u`, `UPDATE … FROM <other db>`, cross-database subqueries, and `CREATE TABLE aux.x AS SELECT … FROM main.y` all work (verified vs upstream). Qualified column references resolve in one, two, or three parts (`col`, `t.col`, `db.t.col`). Transactions span databases: `BEGIN`/`SAVEPOINT` snapshot main, temp, and every attached database, so `ROLLBACK`/`ROLLBACK TO` atomically undo all of them and `COMMIT` persists all file-backed databases together | -| Virtual tables | ⬜ | `unit/test-parse`, `diff/test-write` | `vtab*.test` | `CREATE VIRTUAL TABLE` is an explicit unsupported case; FTS/RTree/etc. remain out of scope for the current storage engine | -| Table-valued functions in FROM | 🟡 | `unit/test-tvf`, `diff/test-json` | `tabfunc*.test`, `json*.test` | `name(args)` in FROM parses to a tvf-ref that resolves to a synthetic scan; `json_each(X[,path])` and `json_tree(X[,path])` (8 SQLite columns; `key`/`value`/`type`/`atom`/`parent`/`fullkey`/`path` match the oracle, `id`/`parent` are a best-effort sequential counter since SQLite's reflect JSONB byte offsets — `json_tree` pre-order DFS with the path-scoped root keeping its key/parent path) `generate_series(start[,stop[,step]])`, and the schema-introspection table-valued PRAGMA functions (`pragma_table_info`/`pragma_table_xinfo`/`pragma_index_list`/`pragma_index_info`/`pragma_index_xinfo`/`pragma_foreign_key_list`, e.g. `SELECT * FROM pragma_table_info('t')`) are implemented; a tvf may be **lateral/correlated** — `SELECT t.id, je.value FROM t, json_each(t.data) je` re-evaluates the function against each outer row (cross/inner joins; outer joins fall back to a single non-correlated evaluation); user/eponymous virtual-table tvfs are not done. Non-reserved keywords (`key`, `value`, `offset`, `match`, ...) are now accepted as column names, matching SQLite's identifier fallback | -| Triggers | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write` | `trigger*.test` | executable subset: schema-stored row-level `BEFORE`/`AFTER INSERT`, `BEFORE`/`AFTER UPDATE [OF ...]`, and `BEFORE`/`AFTER DELETE` triggers on ordinary tables; `INSTEAD OF INSERT`/`UPDATE [OF ...]`/`DELETE` triggers on views; optional `WHEN`, trigger-body INSERT/UPDATE/DELETE/SELECT steps, `NEW`/`OLD` row references, `RAISE(ABORT/FAIL/ROLLBACK, expr)` with trigger constraint errors, `RAISE(IGNORE)` trigger-program/row skipping, and `DROP TRIGGER`; `CREATE TEMP TRIGGER` fires on a temp table or on a table in another schema (a temp trigger on a main/attached table is created and fired via the cross-schema object set), and each trigger-body write step is routed to the schema that holds its target table, so a temp trigger on a main table can log into a temp table; not yet: statement triggers, recursive-compatibility edge cases, or the full trigger-program grammar | -| `DROP TABLE` (+ IF EXISTS) | 🟡 | `diff/test-conflict`, `unit/test-conflict`, `unit/test-constraints`, `diff/test-constraints` | `drop*.test` | rewrites schema; dropped table/index pages go to freelist; with foreign keys enabled, runs SQLite-like implicit DELETE actions for valid incoming references | -| `CREATE INDEX` / `CREATE UNIQUE INDEX` (+ IF NOT EXISTS) | 🟡 | `unit/test-parse`, `unit/test-write`, `unit/test-constraints`, `diff/test-write`, `diff/test-constraints` | `index*.test` | rowid-table column and expression indexes; built-in `main.` qualifier on index name; ASC/DESC and COLLATE; partial-index WHERE predicates; UNIQUE enforced; planner use later | -| `DROP INDEX` (+ IF EXISTS) | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write`, `diff/test-analyze` | `drop*.test` | frees user-index pages; removes matching `sqlite_stat1` rows; upstream-readable file coverage; autoindexes rejected | -| `REINDEX` | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write` | `reindex*.test` | rebuilds all indexes, a named table/index, or indexes using built-in BINARY/NOCASE/RTRIM collations; custom collations later | -| `INSERT ... VALUES` / `DEFAULT VALUES` / `INSERT ... SELECT` (+ insert affinity) | ✅ | `diff/test-write`, `unit/test-write` | `insert*.test` | multi-page; SELECT source materialized before insert; AUTOINCREMENT sequence allocation | -| `INSERT OR IGNORE / OR REPLACE / OR FAIL / OR ROLLBACK`, `REPLACE INTO` | ✅ | `diff/test-conflict`, `unit/test-conflict` | `conflict*.test` | ignore skips violations; replace removes conflicts; FAIL preserves prior row changes in the statement; ROLLBACK aborts the statement in autocommit and rolls back the active transaction; schema-level `ON CONFLICT IGNORE/REPLACE/FAIL/ROLLBACK` policies for rowid/UNIQUE/NOT NULL, including NOT NULL defaults for REPLACE | -| UPSERT (`ON CONFLICT ... DO NOTHING/UPDATE`) | 🟡 | `diff/test-conflict`, `diff/test-returning`, `unit/test-conflict`, `unit/test-parse` | `upsert*.test` | column conflict targets over rowid/UNIQUE constraints, including partial UNIQUE index targets with matching WHERE predicates; `excluded` values, optional DO UPDATE WHERE, and multi-clause target selection; broader edge cases later | -| `UPDATE ... SET ... [FROM] [WHERE]` | ✅ | `diff/test-write`, `diff/test-conflict`, `diff/test-returning`, `unit/test-write`, `unit/test-conflict` | `update*.test` | multi-page rewrite; `UPDATE ... FROM` over tables/joins/CTEs; `OR IGNORE` / `OR REPLACE` / `OR FAIL` / `OR ROLLBACK` for rowid/UNIQUE/NOT NULL/CHECK conflicts; `ORDER BY` + `LIMIT/OFFSET` target selection | -| `DELETE FROM ... [WHERE]` | ✅ | `diff/test-write`, `diff/test-returning`, `unit/test-write` | `delete*.test` | multi-page rewrite; `ORDER BY` + `LIMIT/OFFSET` target selection | -| DML `RETURNING` | 🟡 | `diff/test-returning`, `unit/test-api`, `unit/test-conflict`, `unit/test-parse` | `returning*.test` | `INSERT`/`UPDATE`/`DELETE RETURNING` over target rows, including `*`, `t.*`, scalar expressions, UPSERT affected rows, and UPDATE/DELETE target `ORDER BY` + `LIMIT/OFFSET`; broader edge cases later | -| `ALTER TABLE` | 🟡 | `unit/test-parse`, `unit/test-write`, `diff/test-write`, `diff/test-constraints` | `alter*.test` | `ALTER TABLE ... RENAME TO ...` rewrites table/index schema rows and sqlite_sequence entries; `ADD [COLUMN]` supports ordinary columns with defaults, validates added CHECK constraints, and applies SQLite's REFERENCES/default rule when foreign keys are enabled, while rejecting PK/UNIQUE/generated additions; `RENAME [COLUMN] ... TO ...` rewrites table constraints plus user index column, expression, and partial-predicate references; `DROP [COLUMN]` rewrites ordinary non-dependent columns and rejects indexed/partial-index predicate dependencies; broader dependency rewrites later | -| `NOT NULL` constraint | ✅ | `unit/test-write` | `notnull*.test` | on INSERT + UPDATE; exact errmsg | -| `DEFAULT` column values | ✅ | `diff/test-write`, `unit/test-write` | `default*.test` | literal defaults; affinity applied | -| `UNIQUE` / `PRIMARY KEY` (column, composite, text, integer) | ✅ | `diff/test-constraints`, `unit/test-constraints` | `unique*.test` | enforced by row scan; declared collations honored; autoindexes maintained for integrity_check; NULLs distinct | -| `CHECK` constraints (column + table level) | ✅ | `unit/test-check` | `check*.test` | fails only when expr is false; enforced on INSERT/UPDATE; OR IGNORE skips | -| FOREIGN KEY | 🟡 | `unit/test-constraints`, `diff/test-constraints` | `fkey*.test` | `PRAGMA foreign_keys` and `defer_foreign_keys`; immediate and `DEFERRABLE INITIALLY DEFERRED` timing; pre-existing violations are tolerated until touched; parent-key mismatch validation for referenced PK/UNIQUE keys, including partial-index and collation eligibility; `ON DELETE`/`ON UPDATE`/`DROP TABLE` `RESTRICT`/`CASCADE`/`SET NULL`/`SET DEFAULT`; actions are gated by `PRAGMA foreign_keys`; deeper edge cases later | -| `BEGIN` / `COMMIT` / `END` / `ROLLBACK` / `SAVEPOINT` / `RELEASE` / `ROLLBACK TO` | 🟡 | `unit/test-transaction`, `unit/test-attach`, `diff/test-transaction` | `trans*.test`, `savepoint*.test` | nested image snapshots spanning main, temp, and every attached database (cross-database atomic ROLLBACK/COMMIT); deferred file write until COMMIT/outermost RELEASE, then all file-backed databases persist together; DELETE-mode rollback journal with hot-journal recovery; locking/fsync later | -| On-disk index B-trees | 🟡 | `diff/test-conflict`, `diff/test-write`, `diff/test-select` | — | autoindex + user-created roots maintained through DML; low-level index scans read leaf/interior pages and overflow payloads; multi-level and overflow index writes pass upstream integrity_check; mandatory, simple unhinted, and covered-projection SELECT scans use user-index order; broader planner use later | -| Freelist management | 🟡 | `diff/test-write`, `diff/test-bigtable`, `diff/test-conflict` | — | table rebuild/drop pages are freed and allocator reuses freelist pages | -| `SELECT ... FROM sqlite_master/sqlite_schema` | ✅ | `unit/test-write` | — | implicit schema table | -| PRAGMA (table_info / table_xinfo / table_list / index_list / index_info / index_xinfo / database_list / collation_list / foreign_key_list / foreign_key_check / user_version / application_id / schema_version / data_version / journal_mode / page_count / freelist_count / page_size / cache_size / integrity_check / quick_check / encoding / foreign_keys / defer_foreign_keys / recursive_triggers / trusted_schema / ignore_check_constraints / query_only / count_changes / reverse_unordered_selects / read_uncommitted / automatic_index / busy_timeout / synchronous / temp_store / locking_mode / case_sensitive_like / secure_delete / cell_size_check / fullfsync / checkpoint_fullfsync / journal_size_limit / analysis_limit / optimize / mmap_size / wal_autocheckpoint / threads / function_list / pragma_list / module_list) | 🟡 | `diff/test-pragma`, `unit/test-pragma`, `unit/test-check`, `diff/test-constraints` | `pragma*.test` | introspection + a few settings; `function_list` lists jsqlite's built-in functions (type `s`/`w`), `pragma_list` its PRAGMA names, and `module_list` its eponymous table-valued functions (`json_each`/`json_tree`/`generate_series`) — content is jsqlite-specific (SQLite's per-build `narg`/`flags` are not modelled), so these are unit-tested, and all three are also usable as the `pragma_*` table-valued form; schema-qualified introspection (`PRAGMA temp.table_info`, `PRAGMA <db>.index_list`, ...) resolves the named schema, and unqualified introspection of a table name shadows temp over main then attached; `table_info`/`table_xinfo` cover tables and SELECT/WITH/VALUES/compound views; `table_list` reports main-schema tables/views and built-in schema rows; `index_info`/`index_xinfo` include expression-index rows plus rowid auxiliary columns for rowid-table indexes; `database_list` reports `main`, then `temp` once a temp object exists, then each attached database; `collation_list` reports built-in collations; page/freelist counts read the database image/header; user_version/application_id/schema_version persist; `recursive_triggers`, `trusted_schema`, `ignore_check_constraints`, `query_only`, `count_changes`, `reverse_unordered_selects`, `read_uncommitted`, `automatic_index`, `busy_timeout`, `synchronous`, `temp_store`, `locking_mode`, `case_sensitive_like`, `secure_delete`, `cell_size_check`, `fullfsync`, `checkpoint_fullfsync`, `journal_size_limit`, `analysis_limit`, `mmap_size`, `wal_autocheckpoint`, and `threads` are connection-level flags/settings; no-row `PRAGMA optimize` is accepted with SQLite-shaped columns, but recommendation output such as `optimize(-1)` is not modeled yet; `automatic_index` does not disable ordinary user indexes, matching SQLite; transient automatic indexes are not modeled yet; `query_only` rejects writes with `SQLITE_READONLY`; `count_changes` returns SQLite-shaped affected-row counts for INSERT/UPDATE/DELETE without RETURNING, including view row counts for INSTEAD OF trigger DML while `changes()` remains zero; `reverse_unordered_selects` reverses simple unordered base-table SELECT scans while ORDER BY and VALUES remain stable; `case_sensitive_like` changes LIKE operator/function matching for reads and DML constraint evaluation; `ignore_check_constraints` gates CHECK enforcement and integrity CHECK reporting; `data_version` covers single-connection shape, not cross-connection invalidation; journal_mode/cache_size/synchronous/locking/fullfsync/WAL/checkpoint settings do not yet model full pager locking/fsync/WAL behavior | -| `VACUUM` | 🟡 | `diff/test-vacuum`, `unit/test-parse`, `unit/test-transaction` | `vacuum*.test` | simple `VACUUM [schema]` and `VACUUM INTO` database copies; rejects transaction-time VACUUM and existing output files; physical compaction later | -| `ANALYZE` | 🟡 | `diff/test-analyze`, `unit/test-parse`, `unit/test-transaction` | `analyze*.test` | accepts `ANALYZE`, schema targets, table/index targets, SQLite-like unknown-name errors, and `sqlite_stat1` rows for tables plus ordinary, expression, and partial indexes; broader planner statistics later | -| REAL affinity on read | ✅ | `diff/test-select`, `unit/test-exec` | — | integer-stored reals → REAL | -| `sqlite_schema` read + CREATE TABLE column resolution | ✅ | `unit/test-fileformat` | — | column names + INTEGER PRIMARY KEY rowid alias | - -## API - -| Feature | Status | Diff/unit test | Notes | -|---------|--------|----------------|-------| -| open/open-v2/close | ✅ | `unit/test-api`, `diff/*` | in-memory, ordinary read-write/create files, and read-only file handles | -| prepare/step/reset/finalize | ✅ | `unit/test-api` | lifecycle: before→at-row→done | -| column count/name/type/value | ✅ | `unit/test-api` | | -| bind_* / clear-bindings / bind-parameter-count/name/index | ✅ | `unit/test-api` | bind by index; range-checked; generic and typed helpers; named parameter metadata lookup | -| exec/query helpers | ✅ | `unit/test-api` | step-and-finalize convenience helpers with positional bind args | -| changes / total_changes / last_insert_rowid | ✅ | `unit/test-api` | row-change counters and last inserted rowid | -| errcode / extended_errcode / errmsg | ✅ | `unit/test-api` | connection error state for raised SQLite errors | -| Result/error codes | ✅ | `unit/test-constants`, `unit/test-error` | primary + extended codes | -| Application-defined functions / collations | 🟡 | `unit/test-api` | `sqlite-create-function` registers a scalar SQL function (fixed arity or `-1` variadic; same name+arity replaces); `sqlite-create-aggregate` registers an aggregate (init/step/final), and `sqlite-create-collation` registers a collating sequence usable in `COLLATE`, `ORDER BY`, and column definitions. Per-connection, consulted at prepare (aggregate detection) and execute. the commit/rollback/update hooks (`sqlite-commit-hook`/`sqlite-rollback-hook`/`sqlite-update-hook`; the commit hook can veto a COMMIT into a ROLLBACK; the update hook fires per base-table row change with op/db/table/rowid, including trigger-internal changes). Incremental BLOB I/O (`sqlite-blob-open`/`-read`/`-write`/`-bytes`/`-reopen`/`-close`) addresses bytes within one (schema, table, column, rowid) cell: reads return a sub-range, writes overwrite in place without changing the value's length and persist through a row update, read-only handles refuse writes, and out-of-range access errors. A progress handler (`sqlite-progress-handler` db n proc) is invoked once per `n` scanned B-tree rows and aborts the statement with `SQLITE_INTERRUPT` if it returns true, and `sqlite-interrupt` sets a one-shot flag that aborts the running/next statement the same way. An authorizer (`sqlite-set-authorizer` db proc) is consulted at prepare for each action a statement performs (`SQLITE_SELECT`/`INSERT`/`UPDATE`/`DELETE`/`CREATE_*`/`DROP_*`/`PRAGMA`/`ATTACH`/`DETACH`/`ALTER_TABLE`/`REINDEX`/`ANALYZE`, with table/column/name args); `SQLITE_DENY` rejects the statement with `SQLITE_AUTH`, `SQLITE_OK` allows. Statement-level granularity (column-level `SQLITE_READ` and `SQLITE_IGNORE`-as-NULL are not modelled, and TEMP/attached schemas report as `main`). `sqlite-create-window-function` registers an aggregate usable in an `OVER` clause; jsqlite recomputes the aggregate over each frame, so a user aggregate (or a window function) matches the built-in window aggregates exactly across partitions and sliding frames (the `xInverse`/`xValue` callbacks are accepted but not required). `sqlite-create-module` registers a read-only eponymous virtual-table module: `SELECT ... FROM name(args)` evaluates the args and calls the module's rows function, and the result participates in projection, `WHERE`, aggregation, and joins like any table-valued function. Not yet on the virtual-table side: `CREATE VIRTUAL TABLE` persistence, constraint pushdown (`xBestIndex`), and writable modules. A **C ABI shim** (`src/jsqlite/cshim.ss` + `shim/jsqlite.h`) exposes the core operations as C-callable function pointers — `open`/`prepare`/`step`/`column_count`/`column_int`/`column_text`/`column_type`/`finalize`/`close`/`errmsg`, with connections/statements as opaque integer handles — gathered into a `jsq_api` struct by `build-c-api`; `shim/driver.c` is a C program that drives a full CREATE/INSERT/SELECT cycle through it (`make cshim`), and `unit/test-api` calls the same entry points with the C calling convention. The Chez runtime hosts the engine, so the C side obtains the `jsq_api` from the embedding rather than from a standalone `main` that boots Chez itself. `sqlite-backup` copies a whole database (snapshot, or load a file into memory) | - -## File format - -| Feature | Status | Test | Notes | -|---------|--------|------|-------| -| Header parse (100 bytes) | ✅ | `unit/test-fileformat` | page sizes 512–65536, reserved bytes | -| Varint / record decode | ✅ | `unit/test-record` | serial types, signed ints, float64, text/blob | -| Varint / record encode | ✅ | `unit/test-record` | round-trips; int64 extremes | -| Table B-tree read (interior + leaf) | ✅ | `unit/test-fileformat`, `diff/test-fileread` | full scan, rowid order, rowid point lookup | -| Overflow pages (read) | ✅ | `diff/test-fileread` | exact src/btree.c local-payload math | -| Create DB / CREATE TABLE / INSERT (write) | ✅ | `diff/test-write`, `diff/test-bigtable` | multi-page; verified by upstream integrity_check | -| Multi-page B-tree write (interior + leaves) | ✅ | `diff/test-bigtable`, `unit/test-write` | bottom-up rebuild; 1500+ rows | -| Overflow on write | ✅ | `diff/test-bigtable` | large payloads spill to overflow chain | -| Reserved-bytes pages (write) | ✅ | `diff/test-reserved` | B-tree cell layout, capacities and overflow thresholds honor the per-page reserved-bytes count, so jsqlite writes valid pages to upstream-created databases (verified by upstream integrity_check) | -| Text encoding (UTF-8 / UTF-16) | ✅ | `diff/test-utf16` | reads and writes UTF-8, UTF-16LE, and UTF-16BE databases: records serialize text in the database's declared encoding, and BINARY collation orders text by its encoded bytes (so UTF-16 text indexes and ORDER BY match upstream). `PRAGMA encoding` reports the database's actual encoding, and `PRAGMA encoding='UTF-16le'`/`'UTF-16be'`/`'UTF-8'` on an empty database sets it, so jsqlite can create a UTF-16 database from scratch (upstream-verified) | -| Rollback journal (crash safety) | 🟡 | `diff/test-transaction` | simple DELETE-mode journal and hot-journal restore; no fsync/locking model yet | -| Index B-tree read/write | 🟡 | `diff/test-conflict`, `diff/test-write`, `diff/test-select` | reads leaf/interior index pages, including overflow payloads; autoindex and user-created column/expression roots; multi-level and overflow index writes; mandatory `INDEXED BY`, simple unhinted, and covered-projection SELECT scans can use index order; broader planner selection later | -| WAL (read + write + checkpoint) | 🟡 | `unit/test-wal`, `unit/test-api` | jsqlite reads a database left in WAL mode by merging committed `-wal` frames over the main image when the file is opened: frames are checksum-validated (algorithm verified against upstream output), the latest committed frame wins per page, the database is resized to the commit, and torn/uncommitted frames are ignored. `PRAGMA journal_mode=WAL` switches the connection to WAL mode, after which a commit **appends** the pages it changed as new frames to the `-wal` (continuing the salt and running-checksum chain) rather than rewriting the whole database; the main file is left untouched, and any reader (jsqlite or, after a checkpoint, upstream SQLite) sees the merged state. Because commits append, several jsqlite processes writing one WAL-mode database under the write lock accumulate correctly (the 4-process stress test passes in WAL mode too). The `-wal` is folded back into the main file (checkpoint) when the connection leaves WAL mode or closes — under the write lock, so it cannot race a concurrent writer — leaving an upstream-readable rollback-mode database (`integrity_check` passes). A **shared-memory WAL-index** (`src/jsqlite/shm.ss`) backs WAL coordination: every connection `mmap`s the `<path>-shm` file `MAP_SHARED` (via libc, not libsqlite3), and a writer publishes the WAL salts, committed frame count, page count, a per-commit **generation** number, and a frame→page index into it under the write lock. Other processes read the generation from shared memory to decide when to reload (instead of re-scanning the `-wal`); a checkpoint invalidates the index in place (the file is kept so concurrent mappings observe the change). The layout is jsqlite's own (not byte-compatible with upstream's wal-index), so a *live, uncheckpointed* `-wal` is still not exposed to a concurrent **upstream** process; that, and per-reader snapshots, are the remaining WAL items | -| Multi-process write locking | 🟡 | `unit/test-api`, `tools/concurrency-test` (`make concurrency`) | A writer takes a POSIX `flock` advisory exclusive lock (on a `<path>-lock` sidecar, via libc — not libsqlite3) for its read-modify-write: an autocommit statement locks per statement, an explicit transaction from `BEGIN`/first `SAVEPOINT` through `COMMIT`/`ROLLBACK`. On taking the lock it refreshes the main image to the latest committed state (detected by the header change counter) so writes build on, rather than clobber, concurrent commits. A contended lock retries under `sqlite-busy-timeout` / `sqlite-busy-handler` and otherwise returns `SQLITE_BUSY`. Opening a database (hot-journal recovery + initial load) also holds the lock, so a starting connection cannot mistake another process's in-progress rollback journal for a crash and undo its commit. A 4-process stress test inserts concurrently in both rollback and WAL journal modes with no lost updates, no rowid collisions, and a passing `integrity_check` (`make concurrency`). Locking coordinates jsqlite processes with each other (advisory, keyed on the file); reads use their open-time snapshot (no shared-lock/`-shm` reader-snapshot coordination yet), and attached databases are not separately locked | - -## Known limitations - -- Virtual tables, full trigger support, broader window functions, native WAL - writing (producing `-wal` frames), and index planner use remain planned or - partial as noted above. WAL-mode databases are read (frames merged on open) - and writable: the first jsqlite write checkpoints the merge into the main file - (rollback mode) and removes the `-wal`. -- The write path honors each page's reserved-bytes count, so jsqlite can write - to databases created by upstream SQLite (which currently reserves bytes per - page), not only to databases it created itself. -- TEMP tables/indexes/views and ATTACH/DETACH are supported, including reads that - span databases in one statement (cross-database joins/subqueries), writes whose - source reads another database (INSERT…SELECT, UPDATE…FROM, CTAS), and - transactions that span databases (BEGIN/SAVEPOINT snapshot main, temp, and every - attached database; ROLLBACK/COMMIT are atomic across all of them). -- WITHOUT ROWID tables are supported (PK-keyed index-B-tree storage, file-format - compatible) but always full-scan, as the rowid index planner does not apply. deleted file mode 100644 --- a/vendor/jsqlite/docs/test-plan.md +++ /dev/null @@ -1,53 +0,0 @@ -# Test plan - -Compliance is tracked on three surfaces, each with tests: - -1. **SQL behavior** — same rows, column names, storage classes, errors, side - effects, and PRAGMA/transaction behavior as SQLite 3.54.0. -2. **File format** — databases written by jsqlite open in upstream SQLite and - pass `PRAGMA integrity_check`; upstream databases open and query in jsqlite. -3. **API shape** — `sqlite3_*`-shaped lifecycle and result codes. - -## Five test lanes - -| Lane | Directory | Needs libsqlite3? | Purpose | -|------|-----------|-------------------|---------| -| Unit | `tests/unit/` | no | tokenizer, varint, record codec, affinity, comparison, page parsing, opcodes | -| Differential | `tests/diff/` | yes (oracle) | run SQL on jsqlite + reference, compare normalized output | -| File round-trip | `tests/fileformat/` | yes | upstream-writes/jsqlite-reads and the reverse, + `integrity_check` | -| Upstream | `tests/upstream/` | partial | SQL-centric cases extracted from `../sqlite/test/*.test` (prioritize `e_*.test`) | -| Fuzz / property | (later) | yes | SQL fuzzing vs oracle; B-tree/pager invariants | - -Run with `make unit`, `make diff`, `make test`. The diff/round-trip lanes are -skipped when the `../jerboa-sqlite-ffi` reference checkout is absent. - -## Normalization rules - -The differential oracle (`tools/oracle-runner.ss`) reduces each query to a -normalized result so the two engines can be compared: - -- `(ok <colnames> <rows>)` — rows are lists of lists of jsqlite values. -- `(error <message>)` — any failure. -- `(unsupported <message>)` — jsqlite cannot handle this SQL yet (skipped, not - failed, while features are still being built up). - -Comparison rules: - -- **Preserve storage class**, not just printed value: INTEGER `1` ≠ REAL `1.0`. - This falls out of `sql-value-equal?`, which is storage-class aware. -- Compare column names when the case asserts them. -- Compare **error class / result code before** human-readable message text. - (Phase 0 compares only that both engines reject malformed SQL; result-code - comparison arrives with the real parser in Phase 1.) -- Normalize row order only when the SQL has no `ORDER BY` and the case declares - the output unordered. -- Pin to SQLite 3.54.0; document any version-specific differences explicitly. - -## Acceptance gates (per phase) - -- **Phase 0** — `make test` runs; the oracle compares `SELECT 1`, `SELECT - NULL`, constant literals, and rejects malformed SQL. ✅ -- **Phase 1** — differential pass for constants, operators, `typeof`, `CAST`, - NULL behavior, parameter binding, basic scalar functions; begin pulling cases - from `../sqlite/test/{e_expr,affinity,cast,bind}*.test`. -- Later phases: see `OPUS_4_8_SQLITE_HANDOFF.md`. deleted file mode 100644 --- a/vendor/jsqlite/shim/driver.c +++ /dev/null @@ -1,43 +0,0 @@ -/* driver.c -- a C program that drives jsqlite prepared statements. - * - * `run_demo` is ordinary C against the jsq_api function-pointer table: it opens - * a database, creates and populates a table, then prepares a SELECT and walks - * the rows reading typed columns -- exactly the "drive prepared statements from - * C" path. It returns the sum of the id column so a caller can assert the - * round-trip. The Scheme harness fills the jsq_api (build-c-api), loads this as - * a shared object, and calls run_demo. */ -#include <stdio.h> -#include "jsqlite.h" - -/* prepare + step-to-DONE + finalize a statement that returns no rows */ -static int exec_sql(jsq_api* a, jsq_db db, const char* sql) { - jsq_stmt st = a->prepare(db, sql); - if (!st) { fprintf(stderr, "prepare failed: %s\n", a->errmsg(db)); return -1; } - int rc; - while ((rc = a->step(st)) == JSQ_ROW) { /* discard */ } - a->finalize(st); - return rc; -} - -long long run_demo(jsq_api* a, const char* path) { - jsq_db db = a->open(path); - if (!db) return -1; - - if (exec_sql(a, db, "CREATE TABLE t(id INTEGER PRIMARY KEY, name TEXT)") < 0) { a->close(db); return -2; } - if (exec_sql(a, db, "INSERT INTO t VALUES (1,'alice'),(2,'bob'),(3,'carol')") < 0) { a->close(db); return -3; } - - jsq_stmt st = a->prepare(db, "SELECT id, name FROM t ORDER BY id"); - if (!st) { fprintf(stderr, "prepare failed: %s\n", a->errmsg(db)); a->close(db); return -4; } - - printf("C driver: %d columns per row\n", a->column_count(st)); - long long sum = 0; - while (a->step(st) == JSQ_ROW) { - long long id = a->column_int(st, 0); - const char* name = a->column_text(st, 1); - printf(" row: id=%lld (type %d), name=%s\n", id, a->column_type(st, 0), name ? name : "(null)"); - sum += id; - } - a->finalize(st); - a->close(db); - return sum; -} deleted file mode 100644 --- a/vendor/jsqlite/shim/jsqlite.h +++ /dev/null @@ -1,40 +0,0 @@ -/* jsqlite.h -- C ABI for the jsqlite engine. - * - * jsqlite is implemented in Chez Scheme; this header describes the function - * pointers it exposes to C (see src/jsqlite/cshim.ss). A program obtains a - * filled `jsq_api` from the embedding side (build-c-api) and drives prepared - * statements through it. Connections and statements are opaque integer handles - * (0 means failure / not-a-handle), mirroring sqlite3* / sqlite3_stmt*. - */ -#ifndef JSQLITE_H -#define JSQLITE_H - -typedef long jsq_db; /* opaque connection handle */ -typedef long jsq_stmt; /* opaque statement handle */ - -/* result codes (subset of sqlite3) */ -#define JSQ_OK 0 -#define JSQ_ROW 100 -#define JSQ_DONE 101 - -/* column types (sqlite3_column_type values) */ -#define JSQ_INTEGER 1 -#define JSQ_FLOAT 2 -#define JSQ_TEXT 3 -#define JSQ_BLOB 4 -#define JSQ_NULL 5 - -typedef struct { - jsq_db (*open)(const char* path); /* -> db handle, 0 on error */ - jsq_stmt (*prepare)(jsq_db, const char* sql); /* -> stmt handle, 0 on error */ - int (*step)(jsq_stmt); /* JSQ_ROW / JSQ_DONE / err code */ - int (*column_count)(jsq_stmt); - long long (*column_int)(jsq_stmt, int col); - const char* (*column_text)(jsq_stmt, int col); /* NULL for SQL NULL */ - int (*column_type)(jsq_stmt, int col); - int (*finalize)(jsq_stmt); - int (*close)(jsq_db); - const char* (*errmsg)(jsq_db); -} jsq_api; - -#endif /* JSQLITE_H */ deleted file mode 100644 --- a/vendor/jsqlite/src/jsqlite/api.ss +++ /dev/null @@ -1,2153 +0,0 @@ -#!chezscheme -;;; jsqlite/api.ss -- public database/statement API (Jerboa-native shape). -;;; -;;; open -> prepare -> step -> column-* -> reset/finalize, mirroring sqlite3_*. -;;; Databases are held as a mutable in-memory image (jsqlite/writer): SELECT is -;;; planned/executed by jsqlite/exec; CREATE TABLE / INSERT mutate the image via -;;; jsqlite/dml and (for file databases) are written back to disk. The file -;;; write path uses a simple DELETE-mode rollback journal. - -(library (jsqlite api) - (export - sqlite-open sqlite-open-v2 sqlite-open-bytevector sqlite-db->bytevector - sqlite-close sqlite-db? - sqlite-prepare sqlite-finalize sqlite-reset sqlite-stmt? - sqlite-step - sqlite-bind! sqlite-clear-bindings sqlite-bind-parameter-count - sqlite-bind-parameter-name sqlite-bind-parameter-index - sqlite-bind-null! sqlite-bind-int! sqlite-bind-real! sqlite-bind-text! sqlite-bind-blob! - sqlite-column-count sqlite-column-name sqlite-column-type sqlite-column-value - sqlite-columns sqlite-row - sqlite-exec sqlite-query - sqlite-changes sqlite-total-changes sqlite-last-insert-rowid - sqlite-errcode sqlite-extended-errcode sqlite-errmsg - sqlite-create-function sqlite-create-aggregate sqlite-create-window-function sqlite-create-collation - sqlite-backup sqlite-commit-hook sqlite-rollback-hook sqlite-update-hook - sqlite-blob-open sqlite-blob-close sqlite-blob-bytes - sqlite-blob-read sqlite-blob-write sqlite-blob-reopen - sqlite-progress-handler sqlite-interrupt sqlite-set-authorizer - sqlite-create-module sqlite-busy-handler sqlite-busy-timeout) - - (import (except (chezscheme) - make-hash-table hash-table? - sort sort! - printf fprintf - path-extension path-absolute? - with-input-from-string with-output-to-string - iota 1+ 1- - partition - make-date make-time) - (jerboa prelude clean) - (jsqlite constants) - (jsqlite error) - (jsqlite value) - (jsqlite tokenize) - (jsqlite ast) - (jsqlite parse) - (jsqlite page) - (jsqlite schema) - (jsqlite writer) - (jsqlite wal) - (jsqlite lock) - (jsqlite shm) - (jsqlite eval) - (jsqlite json) - (jsqlite exec) - (jsqlite dml) - (jsqlite pragma)) - - ;; Install the schema-introspection PRAGMA helpers as the resolver exec.ss - ;; uses for table-valued PRAGMA functions (pragma_table_info('t') etc.); - ;; exec.ss cannot import pragma.ss directly without a dml cycle. - (def (pragma-tvf-dispatch pname schema arg) - (cond [(string=? pname "table_info") (pragma-table-info schema arg)] - [(string=? pname "table_xinfo") (pragma-table-xinfo schema arg)] - [(string=? pname "index_list") (pragma-index-list schema arg)] - [(string=? pname "index_info") (pragma-index-info schema arg)] - [(string=? pname "index_xinfo") (pragma-index-xinfo schema arg)] - [(string=? pname "foreign_key_list") (pragma-foreign-key-list schema arg)] - [(string=? pname "function_list") (pragma-function-list)] - [(string=? pname "pragma_list") (pragma-pragma-list)] - [(string=? pname "module_list") (pragma-module-list)] - [else #f])) - - ;; img: mutable database image; schema: cached sqlite_schema objects; - ;; changes/last-rowid: results of the most recent modification. - ;; snapshot: stack of saved images for BEGIN/SAVEPOINT, newest first, or #f - ;; in autocommit mode. While a transaction/savepoint is open, mutations stay - ;; in the image and are NOT written to disk until COMMIT or outermost RELEASE; - ;; ROLLBACK restores the oldest snapshot, ROLLBACK TO restores a named one. - (defstruct sqlite-db-rec (path open? img schema changes total-changes last-rowid - journal-mode foreign-keys defer-foreign-keys cache-size - recursive-triggers trusted-schema ignore-check-constraints query-only count-changes - reverse-unordered-selects - pragma-settings - snapshot readonly errcode extended-errcode errmsg - ;; In-memory TEMP schema: a second database image whose - ;; objects (CREATE TEMP ...) shadow `main` for unqualified - ;; names and are never written to disk. #f until first use. - temp-img temp-schema - ;; ATTACH: list of attached-db records (extra named - ;; databases), searched after temp/main for unqualified names. - attached - ;; application-defined functions/collations/ - ;; aggregates (hashtables keyed by downcased name) - user-funcs user-colls user-aggs - ;; commit/rollback/update callbacks (#f or proc) - commit-hook rollback-hook update-hook - ;; progress handler: call progress-fn every - ;; progress-n scanned rows; interrupted? aborts - ;; the running statement with SQLITE_INTERRUPT. - progress-n progress-fn interrupted? - ;; authorizer callback (#f or a proc consulted at - ;; prepare for each action the statement performs) - authorizer - ;; application-defined virtual-table modules - ;; (hashtable name -> (column-names . rows-fn)) - modules - ;; multi-process write locking: a flock handle on - ;; <path>-lock (lazily opened), whether this - ;; connection currently holds it, and a busy - ;; handler (#f or a proc of the retry count). - lock lock-held? busy-handler - ;; WAL mode: the merged image bytes captured at - ;; the start of the current write unit, so a - ;; commit can append only the pages it changed. - wal-base - ;; shared-memory WAL-index: an mmap handle on