Use real root vendor directory
ober
2429acf3772fb328116bc7bccd3f236bd844f8d5
--- a/llm-harvest/Makefile +++ b/llm-harvest/Makefile @@ -1,6 +1,6 @@ JERBOA_HOME ?= $(HOME)/mine/jerboa SCHEME ?= $(JERBOA_HOME)/.chez/bin/scheme -JSQLITE_DIR ?= $(CURDIR)/vendor/jerboa-sqlite/src +JSQLITE_DIR ?= $(CURDIR)/../vendor/jerboa-sqlite/src SRC_DIR ?= $(CURDIR)/src LIBDIRS = $(SRC_DIR):$(JERBOA_HOME)/lib:$(JSQLITE_DIR) deleted file mode 100644 --- a/llm-harvest/vendor/jerboa-sqlite/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# Transpiled / compiled artifacts -/lib/ -*.sls -*.so -*.dylib -*.wpo -**/.jerbuild-hashes - -# OS / editor -.DS_Store -*~ - -# Local scratch logs (never committed) -/.scratch/ - -# Database sidecar files created at runtime (locking / WAL / rollback journal) -*-lock -*-wal -*-shm -*-journal deleted file mode 100644 --- a/llm-harvest/vendor/jerboa-sqlite/Makefile +++ /dev/null @@ -1,139 +0,0 @@ -# jsqlite -- a SQLite3-compatible database engine in Jerboa Scheme. -# -# jerbuild bundles Chez Scheme + the jerboa stdlib, so building jsqlite needs -# only `jerbuild` on PATH. The differential ("diff") test lane additionally -# needs the jerboa-sqlite reference (a thin FFI over the system libsqlite3), -# checked out as a sibling directory; it is TEST-ONLY (see the working rule in -# OPUS_4_8_SQLITE_HANDOFF.md: never call libsqlite3 from production modules). - -JERBUILD ?= jerbuild -JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) -ifeq ($(JH),) -$(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) -endif - -SRC_DIR := $(CURDIR)/src -TOOLS_DIR := $(CURDIR)/tools -REF_DIR := $(CURDIR)/../jerboa-sqlite -REF_LIB := $(REF_DIR)/lib - -# Production execution: just our source + the jerboa stdlib. -LIBDIRS := $(SRC_DIR):$(JH)/lib -# Differential execution: also the test tools + the reference oracle library. -DIFF_LIBDIRS := $(SRC_DIR):$(TOOLS_DIR):$(REF_LIB):$(JH)/lib - -UNIT_TESTS := $(wildcard tests/unit/*.ss) -DIFF_TESTS := $(wildcard tests/diff/*.ss) - -.PHONY: all build unit diff test oracle fuzz corrupt dmlfuzz concurrency cshim bench robustness clean help -.DEFAULT_GOAL := help - -all: test - -# Compile-check: import every production module (R6RS libraries in .ss files -# are consumed directly via libdirs, so there is no transpile step). -build: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/build-check.ss - -# Unit tests run directly against src/ (jerbuild exec compiles on the fly). -unit: - @fail=0; for t in $(UNIT_TESTS); do \ - echo "== unit: $$t =="; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" $$t || fail=1; \ - done; \ - if [ $$fail -ne 0 ]; then echo "UNIT FAILURES"; exit 1; fi - -# Build the reference oracle (shim + transpiled lib) for differential tests. -oracle: - @if [ ! -d "$(REF_DIR)" ]; then \ - echo "reference $(REF_DIR) not found"; exit 1; fi - $(MAKE) -C $(REF_DIR) build - -# Differential tests compare jsqlite against the reference engine. They are -# skipped (not failed) when the reference checkout is unavailable. -diff: - @if [ ! -d "$(REF_DIR)" ]; then \ - echo "SKIP diff: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP diff: reference build failed"; exit 0; }; \ - fail=0; for t in $(DIFF_TESTS); do \ - echo "== diff: $$t =="; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" $$t || fail=1; \ - done; \ - if [ $$fail -ne 0 ]; then echo "DIFF FAILURES"; exit 1; fi - -test: unit diff - -# Robustness tools (not part of the default gate; longer-running, explicit). -# Override count/seed with `make fuzz N=5000 SEED=7`. -fuzz: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP fuzz: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP fuzz: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/fuzz.ss $(N) $(SEED) - -# Malformed-image corpus test; needs only jsqlite (no oracle). -corrupt: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/corrupt-test.ss $(N) $(SEED) - -# Multi-process write-locking stress test: 4 processes hammer one file, then -# verify no writes were lost. Needs only jsqlite (no oracle). `make concurrency N=40`. -concurrency: - @P=/tmp/jsqlite-concurrency.db; n=$${N:-40}; \ - for mode in "" wal; do \ - echo "== concurrency (journal mode: $${mode:-rollback}) =="; \ - rm -f $$P $$P-lock $$P-journal $$P-wal $$P-shm; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss init $$P; \ - for id in 1 2 3 4; do \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss worker $$id $$n $$P $$mode >/dev/null & \ - done; wait; \ - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/concurrency-test.ss verify $$P 4 $$n || exit 1; \ - done - -# Write-path / B-tree property test vs the oracle (`make dmlfuzz N=2000 SEED=7`). -dmlfuzz: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP dmlfuzz: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP dmlfuzz: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/dml-fuzz.ss $(N) $(SEED) - -# C ABI shim: compile the C driver (shim/driver.c) and have it drive jsqlite -# prepared statements through the jsq_api function-pointer table. Needs a C -# compiler (cc); no oracle. -cshim: - $(JERBUILD) exec --libdirs "$(LIBDIRS)" tools/cshim-test.ss - -# Performance benchmark vs upstream SQLite, using SQLite's own 16-step -# "Database Speed Comparison" workload (docs/benchmarks.md). N is the -# big-transaction insert count (default 1000); `make bench N=2000` for more. -bench: - @if [ ! -d "$(REF_DIR)" ]; then echo "SKIP bench: reference $(REF_DIR) not found"; exit 0; fi; \ - $(MAKE) --no-print-directory -C $(REF_DIR) build >/dev/null || \ - { echo "SKIP bench: reference build failed"; exit 0; }; \ - DYLD_LIBRARY_PATH=$(REF_DIR) LD_LIBRARY_PATH=$(REF_DIR) \ - $(JERBUILD) exec --libdirs "$(DIFF_LIBDIRS)" tools/bench.ss $(N) - -robustness: corrupt dmlfuzz concurrency fuzz - -clean: - rm -rf lib - find . \( -name '*.wpo' -o -name '*.so' \) -delete 2>/dev/null || true - -help: - @echo "jsqlite -- SQLite3-compatible engine in Jerboa Scheme" - @echo "" - @echo " make build Compile-check all production modules" - @echo " make unit Run unit tests (tests/unit/*.ss)" - @echo " make diff Run differential tests vs the jerboa-sqlite oracle" - @echo " make test Run unit + diff (default acceptance gate)" - @echo " make fuzz Differential SQL fuzz vs the oracle (N=, SEED=)" - @echo " make corrupt Malformed-database corpus robustness test (N=, SEED=)" - @echo " make dmlfuzz Write-path / B-tree property test vs the oracle (N=, SEED=)" - @echo " make concurrency Multi-process write-locking stress test (N=)" - @echo " make cshim C ABI shim: a C driver runs jsqlite prepared statements (needs cc)" - @echo " make bench Performance benchmark vs upstream SQLite (N=)" - @echo " make robustness corrupt + dmlfuzz + fuzz" - @echo " make clean Remove build artifacts" deleted file mode 100644 --- a/llm-harvest/vendor/jerboa-sqlite/OPUS_4_8_CONTINUATION_HANDOFF.md +++ /dev/null @@ -1,132 +0,0 @@ -# Opus 4.8 Continuation Handoff - -Date: 2026-05-30 -Branch: `jerbuild` -Repository: `/Users/user/mine/jsqlite` - -## Current State - -This repo has moved substantially beyond the original `OPUS_4_8_SQLITE_HANDOFF.md` -bootstrap plan. Treat the current worktree plus `docs/compatibility-matrix.md` -as the authoritative state. - -The last full gate run before this handoff passed: - -```sh -make build -git diff --check -make test -``` - -The differential lane uses the sibling oracle checkout: - -```text -/Users/user/mine/jerboa-sqlite -``` - -The branch remote is: - -```text -origin git@git.sr.ht:~lisp/jsqlite -``` - -## Major Work Completed Since The Previous Commit - -- Broadened SQL execution substantially: joins, derived tables, recursive CTE - support, compound/aggregate/window coverage, date/time and many scalar - functions, PRAGMAs, RETURNING, UPSERT, generated columns, STRICT tables, - ALTER TABLE variants, triggers, foreign keys, VACUUM, ANALYZE, index B-tree - read/write, and planner/index-scan coverage. -- Added and updated extensive unit and differential tests. New diff files - include: - - `tests/diff/test-analyze.ss` - - `tests/diff/test-date.ss` - - `tests/diff/test-returning.ss` - - `tests/diff/test-strict.ss` - - `tests/diff/test-vacuum.ss` -- Updated `docs/compatibility-matrix.md` to reflect the current support level. -- Recently completed conflict-resolution work: - - `INSERT OR FAIL` and `UPDATE OR FAIL` preserve prior row changes in the - current statement. - - Schema-level `ON CONFLICT FAIL` is supported for rowid/UNIQUE/NOT NULL - paths. - - `INSERT OR ROLLBACK`, `UPDATE OR ROLLBACK`, and schema-level - `ON CONFLICT ROLLBACK` now roll back the active transaction and behave like - statement abort in autocommit. - - Focused coverage is in `tests/unit/test-conflict.ss` and - `tests/diff/test-conflict.ss`. - -## Files Worth Reading First - -- `docs/compatibility-matrix.md` for supported/partial/planned surfaces. -- `src/jsqlite/api.ss` for public API, transactions, error recovery, and - statement execution. -- `src/jsqlite/dml.ss` for DDL/DML, constraints, conflict policy handling, - triggers, foreign keys, generated columns, index maintenance, and ALTER TABLE. -- `src/jsqlite/exec.ss` for SELECT planning/execution, joins, CTEs, windows, - aggregate handling, and EXPLAIN QUERY PLAN. -- `src/jsqlite/parse.ss` and `src/jsqlite/ast.ss` for syntax coverage. -- `tools/oracle-runner.ss` for differential test mechanics. - -## Important Caution - -I was about to investigate one more likely bug, but made no code changes for it -after the user asked for this handoff: - -`RAISE(FAIL)` and statement-level `OR FAIL` preserve statement changes. The API -path for fail-action errors currently calls `write-back!`. Verify whether this -incorrectly persists uncommitted transaction changes to disk before COMMIT. A -good next test is: - -1. Open a file database. -2. Create a table and commit an initial row. -3. Begin a transaction. -4. Run a multi-row `INSERT OR FAIL` that preserves one new row then fails. -5. Run `ROLLBACK`. -6. Reopen the file and compare with upstream SQLite. - -Expected SQLite behavior: after the rollback and reopen, only pre-transaction -rows remain. If jsqlite wrote preserved FAIL changes to disk too early, this -will expose it. Confirm against the oracle before changing implementation. - -Relevant code: - -- `src/jsqlite/api.ss`: `restore-after-statement-error!` -- `src/jsqlite/dml.ss`: `finalize-insert-state!`, `finalize-update-state!`, - `apply-row-conflict-policy!` -- `tests/unit/test-write.ss`: trigger `RAISE(FAIL)` persistence coverage -- `tests/unit/test-conflict.ss`: statement-level FAIL and ROLLBACK coverage - -## Current Git Notes - -Do not commit generated `.scratch/*` logs unless explicitly requested. They are -local scratch artifacts, not source or test assets. - -Everything else in `src/`, `tests/`, `docs/`, `README.md`, and this handoff file -is intended to be committed together. - -## Suggested Next Steps - -1. Verify the suspected transaction/FAIL file-persistence issue above with a - focused unit or differential test. -2. If confirmed, fix the API error-recovery path so FAIL inside an active - transaction preserves the in-memory transaction image but does not write the - database file until COMMIT or outermost RELEASE. -3. Run focused tests: - - ```sh - jerbuild exec --libdirs "$PWD/src:$(jerbuild --jerboa-home)/lib" tests/unit/test-conflict.ss - jerbuild exec --libdirs "$PWD/src:$(jerbuild --jerboa-home)/lib" tests/unit/test-transaction.ss - DYLD_LIBRARY_PATH="$PWD/../jerboa-sqlite" LD_LIBRARY_PATH="$PWD/../jerboa-sqlite" \ - jerbuild exec --libdirs "$PWD/src:$PWD/tools:$PWD/../jerboa-sqlite/lib:$(jerbuild --jerboa-home)/lib" \ - tests/diff/test-conflict.ss - ``` - -4. Then run the acceptance gates: - - ```sh - make build - git diff --check - make test - ``` - deleted file mode 100644 --- a/llm-harvest/vendor/jerboa-sqlite/OPUS_4_8_EMERGENCY_HANDOFF.md +++ /dev/null @@ -1,161 +0,0 @@ -# Opus 4.8 Emergency Handoff - -Date: 2026-05-30 -Repository: `/Users/user/mine/jsqlite` -Branch: `jerbuild` - -## Stop Point - -The user interrupted Codex and explicitly asked to stop working and write this -handoff file. Do not continue implementation until the user asks. - -The latest pushed commit before this handoff remains: - -```text -6c746fd Expand SQLite compatibility surface -``` - -This handoff file itself is untracked unless committed later. - -## Current Worktree State To Expect - -Tracked edits made after commit `6c746fd`: - -- `tests/unit/test-conflict.ss` -- `tests/diff/test-conflict.ss` - -Untracked files expected: - -- `OPUS_4_8_EMERGENCY_HANDOFF.md` -- `.scratch/*` logs from earlier runs - -Do not commit `.scratch/*` unless the user explicitly asks. - -## Work Completed After Resume - -The previous continuation handoff suspected that `OR FAIL` / `RAISE(FAIL)` -might write preserved statement changes to disk while a transaction is still -open. Inspection showed `src/jsqlite/api.ss` already guards file writes: - -```scheme -(def (write-back! db) - (unless (or (in-txn? db) (memory-path? (sqlite-db-rec-path db))) - ...)) -``` - -No source code change was needed. Regression coverage was added instead. - -### Unit Test Added - -File: `tests/unit/test-conflict.ss` - -Added a `suite-fail` test: - -```text -INSERT OR FAIL inside transaction does not persist before rollback -``` - -The test: - -1. Opens `/tmp/jsqlite-unit-conflict-fail-txn.db`. -2. Creates `t(a UNIQUE)`. -3. Inserts committed row `0`. -4. Begins a transaction. -5. Inserts transaction-only row `9`. -6. Runs `INSERT OR FAIL INTO t VALUES (1),(1),(2)` and expects - `SQLITE_CONSTRAINT_UNIQUE`. -7. Confirms same connection sees `0`, `1`, and `9`. -8. Runs `ROLLBACK`. -9. Confirms same connection sees only `0`. -10. Reopens the file and confirms only `0` persisted. - -### Differential Test Added - -File: `tests/diff/test-conflict.ss` - -Added helpers: - -- `oracle-reopen-after` -- `jsqlite-reopen-after` -- `okseq/reopen` - -Added an upstream-oracle comparison in `suite-fail-policy` for: - -```sql -CREATE TABLE t(a UNIQUE); -INSERT INTO t VALUES (0); -BEGIN; -INSERT INTO t VALUES (9); -INSERT OR FAIL INTO t VALUES (1),(1),(2); -ROLLBACK; -``` - -Then both engines reopen the file and run: - -```sql -SELECT a FROM t ORDER BY a; -``` - -Expected result matches upstream SQLite: only row `0`. - -## Verification Already Run - -These passed after the test edits: - -```sh -jerboa_verify tests/unit/test-conflict.ss -jerboa_verify tests/diff/test-conflict.ss -jerbuild exec --libdirs "$PWD/src:$(jerbuild --jerboa-home)/lib" tests/unit/test-conflict.ss -jerbuild exec --libdirs "$PWD/src:$(jerbuild --jerboa-home)/lib" tests/unit/test-transaction.ss -DYLD_LIBRARY_PATH="$PWD/../jerboa-sqlite" LD_LIBRARY_PATH="$PWD/../jerboa-sqlite" \ - jerbuild exec --libdirs "$PWD/src:$PWD/tools:$PWD/../jerboa-sqlite/lib:$(jerbuild --jerboa-home)/lib" \ - tests/diff/test-conflict.ss -make build -git diff --check -make test -``` - -Observed results: - -- `tests/unit/test-conflict.ss`: 67 checks, 0 failures, 0 errors -- `tests/unit/test-transaction.ss`: 26 checks, 0 failures, 0 errors -- `tests/diff/test-conflict.ss`: 26 checks, 0 failures, 0 errors -- `make build`: production modules compiled OK -- `make test`: full suite passed - -## Interrupted Next Step - -After the full suite passed, Codex started looking for the next small gap in -`docs/compatibility-matrix.md`, likely around PRAGMA behavior. The user -interrupted before any useful work was done there. - -Aborted commands may have started but should not have changed the repository: - -```sh -sqlite3 -version && sqlite3 -header -column /tmp/jsqlite-opt-test.db ... -rg -n "parse-pragma|pragma" src/jsqlite/parse.ss | head -40 -``` - -Treat PRAGMA investigation as not started. - -## Suggested Next Action - -If the user asks to continue, first inspect current state: - -```sh -git status --short --branch -git diff -- tests/unit/test-conflict.ss tests/diff/test-conflict.ss -``` - -Then either: - -1. Commit the two conflict regression tests plus this handoff if the user wants - the current progress saved, or -2. Continue from `docs/compatibility-matrix.md` and pick the next explicit - partial/planned surface. - -Do not mark the overall jsqlite goal complete. The original -`OPUS_4_8_SQLITE_HANDOFF.md` target is much broader than the regression work -above, and the compatibility matrix still lists planned/partial areas such as -WITHOUT ROWID, TEMP/ATTACH, virtual tables, WAL, pager locking/fsync, and -broader planner/trigger/PRAGMA edge cases. deleted file mode 100644 --- a/llm-harvest/vendor/jerboa-sqlite/OPUS_4_8_SQLITE_HANDOFF.md +++ /dev/null @@ -1,624 +0,0 @@ -# Jerboa SQLite3 Implementation Handoff - -Audience: Opus 4.8 or another long-running coding agent. - -Goal: build a SQLite3-compatible database engine in Jerboa Scheme, using the -local C SQLite checkout and the existing Jerboa FFI wrapper as reference oracles, -not as implementation dependencies. - -## Current Local Context - -- New workspace: `/Users/user/mine/jsqlite` - - Currently empty except this handoff. - - Not a git repository at inspection time. -- Reference FFI wrapper: `/Users/user/mine/jerboa-sqlite` - - Thin Jerboa/C wrapper over system `libsqlite3`. - - Main files: - - `src/jerboa-sqlite.ss` - - `jerboa_sqlite_shim.c` - - `tests/sqlite-test.ss` - - `Makefile` - - Exposes open/close/exec/prepare/step/reset/finalize/bind/column APIs, - plus `last_insert_rowid`, `changes`, and `errmsg`. - - Use this only for differential tests and oracle generation. -- Upstream SQLite checkout: `/Users/user/mine/sqlite` - - Git branch: `master` - - Git commit seen locally: `3d006d0`, dated 2026-05-29. - - SQLite `VERSION`: `3.54.0` - - Fossil manifest UUID: `c7839f7a1749b38f8a36c06d93a7b59095e91ce753e8dd020de273ca8f73a239` - - Useful source files: - - `src/tokenize.c`: tokenizer behavior. - - `src/parse.y`: Lemon grammar and parser semantic actions. - - `src/vdbe.c`: bytecode engine and opcode comments. - - `src/btree.c`, `src/btreeInt.h`: B-tree implementation. - - `src/pager.c`, `src/wal.c`: rollback journal/WAL/page cache behavior. - - `src/build.c`, `src/expr.c`, `src/select.c`, `src/where*.c`: schema, - expression, SELECT, and planner code generation. - - `src/func.c`, `src/date.c`, `src/json.c`: built-in SQL functions. - - Public tests: `test/` has about 1280 files, including `e_*.test` - requirements-oriented tests, `select*.test`, `where*.test`, `btree*.test`, - `pager*.test`, `pragma*.test`, `corrupt*.test`, `crash*.test`, etc. - -Useful Jerboa standard modules available in the local `jerbuild` cache: - -- `(std binary)`: endian-aware bytevector accessors and packed structs. -- `(std mmap)`: memory-mapped file access and byte-level reads/writes. -- `(std parser)`: small parser-combinator library; useful for early prototypes, - but likely too weak for full SQLite grammar. -- `(std proptest)`: property-based testing. -- `(std mmap-btree)`: existing B-tree-like storage library; use only as - inspiration, not as SQLite page-format storage. - -## Official References - -Use official SQLite documentation first, then local upstream source when docs are -not exact enough. - -- Architecture: https://www.sqlite.org/arch.html -- SQL language surface: https://sqlite.org/lang.html -- Omitted SQL features: https://www.sqlite.org/omitted.html -- Datatypes, affinity, comparison rules: https://www.sqlite.org/datatype3.html -- Database file format: https://www.sqlite.org/fileformat.html -- Bytecode engine overview/opcodes: https://www.sqlite.org/opcode.html -- C API specification: https://www.sqlite.org/c3ref/intro.html and - https://www.sqlite.org/capi3ref.html -- Result/error codes: https://www.sqlite.org/rescode.html -- PRAGMAs: https://www.sqlite.org/pragma.html -- Testing strategy: https://www.sqlite.org/testing.html -- SQL Logic Test: https://www.sqlite.org/sqllogictest/doc/trunk/about.wiki -- Requirements process: https://www.sqlite.org/requirements.html - -Important research facts: - -- SQLite's core architecture is SQL text -> tokenizer -> parser -> code - generator -> VDBE bytecode -> B-tree -> pager -> OS/VFS. -- The on-disk SQLite3 database format is stable and specified in detail. A valid - database begins with `SQLite format 3\000`, uses fixed-size pages, stores - schema objects in `sqlite_schema`, and stores table/index content in B-trees. -- SQLite's typing model is value-typed and affinity-driven, not rigidly - column-typed, except for newer `STRICT` table behavior. -- SQLite's documented behavior is intentionally detailed enough to support a - compatible reimplementation. Treat the docs and public tests as the primary - spec. -- TH3 is proprietary. Public validation should lean on upstream Tcl tests, - differential testing through `jerboa-sqlite`, sqllogictest, fuzzing, and - file-format round trips. -- VDBE opcodes are not a public API and can change by release. If this - implementation mirrors VDBE, pin behavior to the local SQLite 3.54.0 checkout - and generate opcode metadata from local source comments/scripts. - -## Definition Of Compliance - -Do not let "fully SQLite3 compliant" remain vague. Track three separate surfaces: - -1. SQL behavior compatibility: - Same rows, column names, storage classes, errors, side effects, trigger - behavior, transaction behavior, and PRAGMA behavior for the supported SQLite - version. - -2. File-format compatibility: - Databases written by Jerboa can be opened by upstream SQLite and pass - `PRAGMA integrity_check`; databases written by upstream SQLite can be opened, - queried, and eventually modified by Jerboa without corruption. - -3. API compatibility: - The Jerboa-native API should be shaped like `sqlite3_open`, - `sqlite3_prepare_v2`, `sqlite3_step`, `sqlite3_column_*`, - `sqlite3_bind_*`, `sqlite3_reset`, `sqlite3_finalize`, `sqlite3_exec`, - `sqlite3_errmsg`, result codes, and related high-use APIs. A real C ABI shim - can come later if needed. - -Initial compatibility target: - -- Pin to local upstream SQLite `VERSION` 3.54.0/trunk as checked out in - `/Users/user/mine/sqlite`. -- Build a compatibility ladder. "Done" is not realistic until public tests, - sqllogictest, crash/corruption tests, and file round-trips are passing. -- Defer optional extension modules at first: FTS3/4/5, RTree/geopoly, session, - ICU, expert, RBU, recover, CSV, extension loading, and arbitrary virtual table - modules. Keep the architecture open for them. - -## Recommended Architecture - -Use SQLite's architecture rather than a direct SQL interpreter. Early prototypes -can run directly from AST, but full compatibility is easier with a VDBE-like -prepared-statement boundary. - -Suggested modules: - -```text -src/jsqlite/constants.ss SQLite result codes, open flags, type tags -src/jsqlite/error.ss condition types, primary/extended result codes -src/jsqlite/value.ss NULL/integer/real/text/blob values, conversions -src/jsqlite/encoding.ss UTF-8 first, UTF-16 later -src/jsqlite/collation.ss BINARY, NOCASE, RTRIM; custom collations later -src/jsqlite/tokenize.ss SQLite tokenizer -src/jsqlite/parse.ss generated or hand-built parser frontend -src/jsqlite/ast.ss SQL AST and schema AST records -src/jsqlite/schema.ss sqlite_schema cache, object resolution -src/jsqlite/record.ss SQLite record varints and serial types -src/jsqlite/page.ss database header and page parsing/writing -src/jsqlite/btree.ss table/index B-tree cursors and mutations -src/jsqlite/pager.ss page cache, rollback journal, locks -src/jsqlite/wal.ss WAL mode, later phase -src/jsqlite/vdbe.ss instruction representation and VM loop -src/jsqlite/codegen.ss AST -> bytecode -src/jsqlite/expr.ss expression analysis/codegen/eval helpers -src/jsqlite/planner.ss WHERE/index selection -src/jsqlite/functions.ss scalar/aggregate/window functions -src/jsqlite/pragma.ss PRAGMA handlers -src/jsqlite/api.ss public database/statement API -src/jsqlite/oracle.ss test-only reference runner via jerboa-sqlite -``` - -Test/tooling layout: - -```text -Makefile -README.md -tests/unit/*.ss -tests/diff/*.ss -tests/fileformat/*.ss -tests/upstream/*.ss -tools/oracle-runner.ss -tools/slt-runner.ss -tools/tcltest-extract.ss -tools/gen-keywords.ss -tools/gen-grammar.ss -docs/compatibility-matrix.md -docs/test-plan.md -docs/architecture.md -``` - -## Core Design Notes - -### Public API - -Start with a Jerboa-native API, not a C ABI: - -- `sqlite-open`, `sqlite-open-v2`, `sqlite-close` -- `sqlite-prepare`, `sqlite-step`, `sqlite-reset`, `sqlite-finalize` -- `sqlite-bind!`, typed bind helpers, `sqlite-clear-bindings` -- `sqlite-column-count`, `sqlite-column-name`, `sqlite-column-type`, - `sqlite-column-value` -- `sqlite-exec`, `sqlite-query` -- `sqlite-last-insert-rowid`, `sqlite-changes`, `sqlite-total-changes` -- `sqlite-errcode`, `sqlite-extended-errcode`, `sqlite-errmsg` - -Later, add a C shim exposing a subset of `sqlite3_*` for applications that -expect the C API. - -### Values And Typing - -Implement this before any serious SQL: - -- Storage classes: NULL, INTEGER signed 64-bit, REAL binary64, TEXT, BLOB. -- SQLite truthiness and three-valued logic. -- Numeric conversion and comparison rules. -- Column affinity rules and expression affinity. -- `typeof`, `CAST`, `IS`, `IS NOT`, `IS DISTINCT FROM`, `IN`, `BETWEEN`. -- Collation precedence and default `BINARY` collation. -- Exact error/result code behavior for constraint and type errors. - -This is a high-risk area because many tests depend on SQLite's quirks. - -### Tokenizer - -Use a hand-coded scanner. Parser combinators are acceptable for experiments, -but SQLite tokenization has enough edge cases that a scanner matching -`src/tokenize.c` is the right long-term path. - -Required token behavior: - -- Whitespace and comments, including SQL comments. -- Identifiers, quoted identifiers, bracket/backtick forms. -- Keywords with fallback-to-ID behavior. -- String literals, doubled quotes, blob literals `x'...'`. -- Numeric literals, including integer/real/hex cases. -- Bind parameters: `?`, `?NNN`, `:name`, `@name`, `$name`. -- Operators and compound tokens. - -Use `tool/mkkeywordhash.c` and `src/parse.y` as references for the keyword set. - -### Parser - -Full compliance needs SQLite's grammar shape. Recommended path: - -1. Early bootstrap: implement a small parser for `SELECT` constants, `CREATE - TABLE`, `INSERT`, simple `SELECT ... FROM ... WHERE ...`, and transaction - statements so other layers can be tested. -2. Parallel long-term path: build or adapt a Lemon-compatible parser generator - in Jerboa, or generate parser tables from `src/parse.y` using a small tool. -3. Translate parser semantic actions into construction of Jerboa AST nodes, not - C structs. - -Do not attempt to support the full `parse.y` grammar manually by accretion -unless this is explicitly accepted as a long-term maintenance burden. - -### VDBE - -Implement a VDBE-like register machine: - -- Instruction record with opcode, P1, P2, P3, P4, P5, comment/debug fields. -- Register vector of `sqlite-value` cells plus undefined state. -- Program counter, halted/error state, row-yield state. -- Statement lifecycle: prepare -> step returns `SQLITE_ROW`/`SQLITE_DONE` -> - reset/finalize. -- Cursors for table/index B-trees, ephemeral tables, sorters, pseudo tables. -- Subprograms for triggers and coroutines for subqueries. - -Start with a minimal opcode set: - -- Control: `Init`, `Goto`, `Halt`, `ResultRow`, `Once`, `If`, `IfNot`, - `IsNull`, `NotNull`. -- Values: `Null`, `Integer`, `Int64`, `Real`, `String8`, `Blob`, `Variable`, - `Copy`, `Move`, `SCopy`, `Cast`, `Affinity`. -- Expressions: arithmetic, concat, comparisons, boolean ops, `Function`. -- Tables: `OpenRead`, `OpenWrite`, `Rewind`, `Next`, `Column`, `Rowid`, - `MakeRecord`, `Insert`, `Delete`. -- Transactions: `Transaction`, `AutoCommit`, `Savepoint`, `Rollback`, - `ReadCookie`, `SetCookie`. - -Use local `src/vdbe.c` comments and generated metadata as source of truth for -opcode behavior. Exact opcode names/numbers only matter for `EXPLAIN` and -debugging, but the prepared-statement lifecycle matters everywhere. - -### File Format And B-Tree - -Build read-only before write support. - -Read path: - -- Parse database header fields from the first 100 bytes. -- Support page sizes 512 through 65536. -- Decode varints and record serial types. -- Read table leaf cells, table interior cells, index leaf/interior cells. -- Follow overflow pages. -- Parse `sqlite_schema` and cache schema objects. -- Support rowid tables first; `WITHOUT ROWID` later. - -Write path: - -- Allocate/free pages and maintain freelist. -- Insert table rows and index entries. -- Split/rebalance B-tree pages exactly enough to keep upstream SQLite happy. -- Maintain schema cookie, change counter, page count, root pages. -- Enforce `INTEGER PRIMARY KEY`, `AUTOINCREMENT`, and `sqlite_sequence`. -- Write valid rollback-journal databases first. WAL comes later. - -Always verify written files with upstream SQLite: - -```sql -PRAGMA integrity_check; -SELECT * FROM sqlite_schema; -``` - -### Pager, Transactions, Locking - -Pager correctness is the hardest non-SQL part. Avoid shortcuts once writes -begin. - -Implementation stages: - -1. Read-only pager and page cache. -2. Single-process write transactions with rollback journal. -3. Hot journal detection and recovery. -4. Savepoints and statement journals. -5. POSIX file locking with SQLite-compatible lock states. -6. Multi-process read/write behavior. -7. WAL mode and checkpoints. - -Until rollback recovery is implemented, mark any write implementation as -experimental and do not claim file-format compliance. - -### Query Planner - -Semantics first, performance second. - -Initial planner: - -- Full table scans. -- Simple nested-loop joins. -- ORDER BY via in-memory sorter. -- GROUP BY/aggregates via in-memory hash/sort. - -Then add: - -- Rowid lookup. -- Single-column index lookup. -- Multi-column index constraints. -- Covering indexes. -- Partial/expression indexes. -- Automatic indexes. -- `sqlite_stat1`/`stat4` and cost estimates. - -Planner output does not need to match upstream exactly unless tests assert -`EXPLAIN QUERY PLAN`. Query results and side effects do. - -## Milestones And Acceptance Gates - -### Phase 0: Scaffold And Oracle Harness - -Deliverables: - -- `Makefile` with `build`, `test`, `unit`, `diff`, `clean`. -- Jerboa library structure under `src/jsqlite`. -- Basic test runner. -- Oracle runner that can execute the same SQL against: - - local Jerboa implementation - - `/Users/user/mine/jerboa-sqlite` wrapper/reference -- Normalized comparison format for rows, storage classes, column names, and - result codes. -- Compatibility matrix document. - -Acceptance: - -- `make test` runs. -- Oracle can compare `SELECT 1`, `SELECT NULL`, simple arithmetic, and expected - syntax errors, even if the Jerboa side initially skips many cases. - -### Phase 1: Values, Tokenizer, Minimal Parser, In-Memory VM - -Deliverables: - -- Value representation and conversions. -- Tokenizer with focused tests. -- Minimal parser for expressions and simple `SELECT` without tables. -- Minimal VDBE or direct evaluator with statement lifecycle. - -Acceptance: - -- Differential tests pass for constants, operators, `typeof`, `CAST`, NULL - behavior, parameter binding, and basic built-in scalar functions. -- Start pulling cases from `../sqlite/test/e_expr.test`, `affinity*.test`, - `cast.test`, `bind*.test`. - -### Phase 2: SQLite File Reader - -Deliverables: - -- Header/page parser. -- Varint and record decoder. -- `sqlite_schema` reader. -- Table B-tree scan cursor. - -Acceptance: - -- Upstream SQLite creates databases; Jerboa reads schemas and table rows. -- Support common page sizes and overflow records. -- Differential read tests pass for simple `CREATE TABLE`/`INSERT` databases. - -### Phase 3: Read-Only SELECT Over Real Tables - -Deliverables: - -- Schema resolution. -- Table cursors in VM. -- WHERE filters, projection, ORDER BY, LIMIT/OFFSET. -- Simple joins. - -Acceptance: - -- Pass a growing subset of `select*.test`, `where*.test`, `e_select*.test`. -- Query results match upstream for row values and storage classes. - -### Phase 4: Basic Writes And Rollback Journal - -Deliverables: - -- Page allocation/free. -- Table B-tree insertion/deletion/update. -- `CREATE TABLE`, `DROP TABLE`, `INSERT`, `UPDATE`, `DELETE`. -- Rollback journal for atomic commit/rollback. - -Acceptance: - -- Files written by Jerboa pass upstream `PRAGMA integrity_check`. -- Upstream can reopen and query Jerboa-written databases. -- Basic rollback works after simulated mid-transaction failures. - -### Phase 5: Indexes, Constraints, And Better Planner - -Deliverables: - -- Index B-trees. -- UNIQUE/PRIMARY KEY/CHECK/NOT NULL/default constraints. -- Conflict resolution: ROLLBACK, ABORT, FAIL, IGNORE, REPLACE. -- Rowid lookup and simple index planner. - -Acceptance: - -- Pass subsets of `insert*.test`, `delete*.test`, `update*.test`, - `conflict*.test`, `index*.test`, `where*.test`. -- Constraint error codes align with upstream. - -### Phase 6: Transactions, Savepoints, Crash Safety - -Deliverables: - -- Autocommit semantics. -- BEGIN DEFERRED/IMMEDIATE/EXCLUSIVE. -- COMMIT/ROLLBACK. -- SAVEPOINT/RELEASE/ROLLBACK TO. -- Hot journal recovery. -- Same-process locking semantics. - -Acceptance: - -- Pass transaction/savepoint/pager-focused tests. -- Crash-simulation harness demonstrates no corrupt committed database. - -### Phase 7: Full SQL Core - -Deliverables: - -- Aggregates, GROUP BY, HAVING. -- Compound SELECT. -- Subqueries and correlated subqueries. -- CTEs including recursive CTEs. -- Views and triggers. -- Foreign keys. -- UPSERT and RETURNING. -- ALTER TABLE variants supported by SQLite. -- Generated columns, STRICT tables, WITHOUT ROWID tables. -- Window functions. - -Acceptance: - -- Large public upstream test subsets pass. -- Sqllogictest starts passing broad generated workloads. - -### Phase 8: PRAGMAs, ATTACH, TEMP, VACUUM, ANALYZE - -Deliverables: - -- High-use PRAGMAs first: `table_info`, `index_list`, `journal_mode`, - `foreign_keys`, `integrity_check`, `user_version`, `application_id`, - `page_size`, `cache_size`, `encoding`. -- TEMP schema. -- ATTACH/DETACH. -- VACUUM. -- ANALYZE and statistics tables. - -Acceptance: - -- Pass `pragma*.test`, `attach*.test`, `vacuum*.test`, `analyze*.test` subsets. - -### Phase 9: WAL And Concurrent Access - -Deliverables: - -- WAL file reader/writer. -- Shared-memory WAL-index equivalent. -- Checkpointing. -- Reader snapshots. -- Busy handling/timeouts. -- Multi-process stress tests. - -Acceptance: - -- Pass WAL-focused tests and multi-process smoke tests. - -### Phase 10: C API Surface And Extensions - -Deliverables: - -- More complete `sqlite3_*` behavior. -- Optional C ABI shim. -- Application-defined functions/collations. -- Authorizer/progress/update hooks. -- Backup/blob APIs. -- Virtual table API. -- Extension loading model, if desired. - -Acceptance: - -- API tests from upstream public suite pass where applicable. -- The C shim can drive prepared statements from C code for core operations. - -### Phase 11: Fuzzing, Corruption, Performance - -Deliverables: - -- SQL fuzzing against oracle. -- Malformed database corpus tests. -- Property tests for B-tree and pager invariants. -- Performance benchmarks against upstream SQLite on representative workloads. - -Acceptance: