Fix musl build and prevent recurring drift breakage
ober
5433a3db8148d27c9d11f236659975af02be6360
--- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ TUI_SHIM_DIR := $(CURDIR)/vendor/termbox2 NATIVE_LIB_DIR := $(JERBOA_HOME)/lib LDPATH := $(SHIM_DIR):$(TUI_SHIM_DIR):$(SQLITE_LIB_DIR):$(NATIVE_LIB_DIR) -.PHONY: all build gen run test clean repl binary install tui-shim run-tui jcode-musl linux linux-local docker +.PHONY: all build gen run test clean repl binary install tui-shim run-tui jcode-musl linux linux-check linux-local docker all: build @@ -57,7 +57,20 @@ install: binary cp jcode $(HOME)/.local/bin/jcode @echo "Installed to ~/.local/bin/jcode" -linux: docker +linux: linux-check docker + +# ── Fast local drift check for the musl build path ───────────────────────── +# Runs compile-program on main-binary.ss with the same flags the musl build +# uses, but skips sed patching, the C compile, and the linker. Catches drift +# (new imports, removed stdlib exports, broken except clauses) on the dev's +# laptop in ~10 seconds, before burning 5+ minutes in Docker. +# +# `linux` and `linux-local` both depend on this target so they fail fast. + +linux-check: gen + @echo "=== Running linux-check (fast musl-build drift check) ===" + DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ + $(SCHEME) -q $(LIBDIRS) --script linux-check.ss # ── Docker build (canonical static binary, zero runtime deps) ─────────────── # Use `make linux` to build in Docker (canonical, reproducible). @@ -80,7 +93,7 @@ docker: # Requires: musl-gcc, Chez Scheme built with musl (~/chez-musl), # jerboa-native-rs built for x86_64-unknown-linux-musl. -linux-local: gen +linux-local: linux-check @echo "=== Building static jcode with musl ===" JERBOA_HOME=$(JERBOA_HOME) ./build-jcode-musl.sh --- a/build-jcode-musl.ss +++ b/build-jcode-musl.ss @@ -50,37 +50,74 @@ (printf "Jerboa lib: ~a~n" jerboa-dir) -;; ========== Module list ========== +;; ========== Auto-discovery helpers ========== +;; +;; These helpers eliminate the hardcoded parallel lists of modules that +;; historically drifted every time a new src/jcode/*.ss file was added. +;; jcode-modules and external-libs are now derived from the filesystem. + +(define (find-files-with-suffix root suffix) + ;; Recursively find all files under `root` whose name ends in `suffix`. + ;; Returns an alphabetically sorted list of full paths. + (if (and (string? root) + (file-exists? root) + (file-directory? root)) + (let loop ([dirs (list root)] [acc '()]) + (if (null? dirs) + (sort string<? acc) + (let* ([d (car dirs)] + [entries (map (lambda (e) (format "~a/~a" d e)) + (directory-list d))] + [files (filter + (lambda (p) + (and (not (file-directory? p)) + (let ([n (string-length p)] + [sn (string-length suffix)]) + (and (>= n sn) + (string=? + suffix + (substring p (- n sn) n)))))) + entries)] + [subdirs (filter file-directory? entries)]) + (loop (append subdirs (cdr dirs)) (append files acc))))) + '())) + +(define (strip-suffix s suffix) + (let ([ls (string-length s)] [lsu (string-length suffix)]) + (if (and (>= ls lsu) + (string=? suffix (substring s (- ls lsu) ls))) + (substring s 0 (- ls lsu)) + s))) + +(define (assert-so-files-exist! who paths kind) + ;; Strict replacement for the old silent `existing-so-files` filter. + ;; Errors loudly if any entry is missing so drift surfaces immediately. + (let ([missing (filter (lambda (p) (not (file-exists? p))) paths)]) + (cond + [(null? missing) paths] + [else + (printf "~n[ERROR] ~a: ~a of ~a ~a .so file(s) missing:~n" + who (length missing) (length paths) kind) + (for-each (lambda (p) (printf " - ~a~n" p)) missing) + (error who + (format "missing ~a .so file(s) — drift between source tree and build script" + kind))]))) + +;; ========== Auto-discover jcode modules ========== +;; lib/jcode/**/*.sls is generated by jerbuild (make gen) before this script +;; runs. Walking the filesystem keeps this list perpetually in sync with the +;; source tree. (define jcode-modules - '("lib/jcode/core/models" - "lib/jcode/core/config" - "lib/jcode/core/log" - "lib/jcode/core/session" - "lib/jcode/core/message" - "lib/jcode/core/agent" - "lib/jcode/core/plugin" - "lib/jcode/provider/provider" - "lib/jcode/tool/registry" - "lib/jcode/tool/file" - "lib/jcode/tool/bash" - "lib/jcode/tool/web" - "lib/jcode/tool/batch" - "lib/jcode/tool/git" - "lib/jcode/tool/lsp" - "lib/jcode/mcp/client" - "lib/jcode/ui/tui-ffi" - "lib/jcode/ui/tui-theme" - "lib/jcode/ui/tui-keys" - "lib/jcode/ui/tui-markdown" - "lib/jcode/ui/tui-diff" - "lib/jcode/ui/tui-message" - "lib/jcode/ui/tui-status" - "lib/jcode/ui/tui-input" - "lib/jcode/ui/tui-sidebar" - "lib/jcode/ui/tui-dialog" - "lib/jcode/ui/tui" - "lib/jcode/ui/cli")) + (map (lambda (sls-path) (strip-suffix sls-path ".sls")) + (find-files-with-suffix "lib/jcode" ".sls"))) + +(printf "~n[discovery] Found ~a jcode modules under lib/jcode~n" + (length jcode-modules)) +(when (null? jcode-modules) + (error 'build-jcode-musl + "No .sls files found under lib/jcode — run 'make gen' first")) +(for-each (lambda (m) (printf " ~a~n" m)) jcode-modules) ;; libc symbols needed by std/net/tcp (already linked in from musl) (define libc-symbols @@ -174,75 +211,76 @@ (generate-inspector-information #f)) (compile-program "main-binary.ss")) +;; Fail loud if compile-program didn't actually produce main-binary.so. +;; Historically this failure was silent — the script would continue until +;; step 5 died with a cryptic "open-file-input-port: failed for main-binary.so". +(unless (file-exists? "main-binary.so") + (printf "~n[ERROR] compile-program completed but main-binary.so was not produced.~n") + (printf " cwd: ~a~n" (current-directory)) + (printf " entry: main-binary.ss~n") + (printf " This usually means an import in main-binary.ss could not be resolved~n") + (printf " or a .sls file was corrupted by the blanket sed patching in step 0.5.~n") + (error 'build-jcode-musl + "main-binary.so not produced by compile-program")) +(printf " → main-binary.so (~a bytes)~n" + (file-length (open-file-input-port "main-binary.so"))) + ;; ========== Step 2: Skip WPO for musl builds ========== ;; Use the direct main-binary.so from compile-program. (printf "[2/7] Skipping WPO (using main-binary.so directly)...~n") (define program-so "main-binary.so") -;; ========== Step 3: Pre-compile boot-file dependencies ========== - -(printf "[3/7] Pre-compiling boot dependencies...~n") -(let ([boot-jerboa-modules - '("jerboa/core" "jerboa/runtime" - "std/error" "std/format" "std/sort" "std/pregexp" "std/sugar" - "std/typed" - "std/misc/string" "std/misc/list" "std/misc/alist" "std/misc/thread" - "std/misc/ports" "std/misc/retry" "std/misc/uuid" - "std/os/path" "std/os/shell" - "std/text/json" "std/text/glob" - "std/net/tcp" "std/net/tls-rustls" "std/net/request" - "std/db/sqlite")]) - (parameterize ([compile-imported-libraries #t] - [optimize-level 2] - [generate-inspector-information #f]) - (for-each - (lambda (m) - (let ([sls (format "~a/~a.sls" jerboa-dir m)] - [so (format "~a/~a.so" jerboa-dir m)]) - (when (and (file-exists? sls) (not (file-exists? so))) - (printf " Pre-compiling ~a~n" sls) - (compile-library sls)))) - boot-jerboa-modules))) +;; ========== Step 3: Auto-discover external libs ========== +;; +;; Historically this step held a hardcoded list of ~22 (std ...) modules to +;; pre-compile, and step 4 held a *second* parallel list of the same modules +;; to embed in the boot file. Both drifted every time jcode imported a new +;; stdlib module, and the failure mode was silent (missing entries were +;; quietly filtered out by `existing-so-files`). +;; +;; Instead: compile-program in step 1 already traversed main-binary.ss's full +;; import graph with compile-imported-libraries #t, emitting a .so next to +;; every imported .sls in $JERBOA_HOME/lib. Step 0.5 wiped that directory +;; clean beforehand, so every .so that now exists there is one the import +;; graph actually needs. Scan the directory to get the authoritative list. + +(printf "[3/7] Discovering external libs compiled under ~a...~n" jerboa-dir) + +(define external-lib-paths + (find-files-with-suffix jerboa-dir ".so")) + +(printf " Found ~a external lib .so file(s)~n" (length external-lib-paths)) +(when (null? external-lib-paths) + (error 'build-jcode-musl + (format "No .so files under ~a after compile-program — did step 1 fail silently?" + jerboa-dir))) ;; ========== Step 4: Create libs-only boot file ========== (printf "[4/7] Creating libs-only boot file...~n") -(define external-libs - (map (lambda (m) (format "~a/~a.so" jerboa-dir m)) - '("jerboa/core" - "jerboa/runtime" - "std/error" - "std/format" - "std/sort" - "std/pregexp" - "std/sugar" - "std/typed" - "std/misc/string" - "std/misc/list" - "std/misc/alist" - "std/misc/thread" - "std/misc/ports" - "std/misc/retry" - "std/misc/uuid" - "std/os/path" - "std/os/shell" - "std/text/json" - "std/text/glob" - "std/net/tcp" - "std/net/tls-rustls" - "std/net/request" - "std/db/sqlite"))) - -(define (existing-so-files paths) - (filter file-exists? paths)) +(define chez-sqlite-so "vendor/chez-sqlite/src/chez-sqlite.so") +(unless (file-exists? chez-sqlite-so) + (error 'build-jcode-musl + (format "chez-sqlite .so missing at ~a — was jerbuild run on vendor/?" chez-sqlite-so))) + +;; jcode module .so files live next to their .sls (same dir), so just swap +;; the suffix. Strict assertion surfaces any missing entry loudly. +(define jcode-so-files + (assert-so-files-exist! 'make-boot-file + (map (lambda (m) (format "~a.so" m)) jcode-modules) + "jcode module")) + +(printf " Boot file contents:~n") +(printf " ~a external lib .so files~n" (length external-lib-paths)) +(printf " 1 chez-sqlite shim .so~n") +(printf " ~a jcode module .so files~n" (length jcode-so-files)) (apply make-boot-file "jcode.boot" '("scheme" "petite") (append - (existing-so-files external-libs) - (list "vendor/chez-sqlite/src/chez-sqlite.so") - (existing-so-files - (map (lambda (m) (format "~a.so" m)) jcode-modules)))) + external-lib-paths + (list chez-sqlite-so) + jcode-so-files)) ;; ========== Step 5: Generate C with embedded boot files + program ========== --- a/lib/jcode/provider/provider.sls +++ b/lib/jcode/provider/provider.sls @@ -9,10 +9,10 @@ (except (chezscheme) make-hash-table hash-table? iota \x31;+ \x31;- getenv path-extension path-absolute? thread? make-mutex mutex? mutex-name) - (std text json) (except (std net request) http-post-stream) - (std net tls-rustls) (std net tcp) (std misc string) - (std misc retry) (jcode core log) (jcode core message) - (jerboa core) (jerboa runtime)) + (std text json) (std net request) (std net tls-rustls) + (std net tcp) (std misc string) (std misc retry) + (jcode core log) (jcode core message) (jerboa core) + (jerboa runtime)) (def logger (make-logger "provider")) (def *api-retry-policy* (make-retry-policy 3 1.0 30.0 #t)) (def (retryable-error? e) new file mode 100644 --- /dev/null +++ b/linux-check.ss @@ -0,0 +1,136 @@ +#!chezscheme +;;; linux-check.ss — Fast local sanity check for the musl static build path. +;;; +;;; Runs compile-program on main-binary.ss with the same flags the musl +;;; build uses, but WITHOUT the Docker round-trip, the sed patching of +;;; jerboa lib, or the musl-gcc linking. Completes in ~10 seconds. +;;; +;;; Catches the most common drift failures on a dev laptop before an +;;; expensive Docker build fails minutes in: +;;; +;;; - A new src/jcode/*.ss file has a broken import +;;; - main-binary.ss references a removed symbol +;;; - A `(except (std ...) foo)` clause refers to a now-unexported foo +;;; (this is how the `http-post-stream` regressions happened repeatedly) +;;; - A required jcode module .so did not get produced +;;; +;;; Usage (from the Makefile, which sets LD_LIBRARY_PATH for native libs): +;;; make linux-check +;;; +;;; Direct: +;;; scheme -q --libdirs "$JERBOA_HOME/lib:./lib:vendor/chez-sqlite/src" \ +;;; --script linux-check.ss + +(import (chezscheme)) + +(printf "~n=== linux-check: fast musl build sanity check ===~n~n") + +;; ---------- helpers ---------- + +(define (find-files-with-suffix root suffix) + (if (and (string? root) (file-exists? root) (file-directory? root)) + (let loop ([dirs (list root)] [acc '()]) + (if (null? dirs) + (sort string<? acc) + (let* ([d (car dirs)] + [entries (map (lambda (e) (format "~a/~a" d e)) + (directory-list d))] + [files (filter + (lambda (p) + (and (not (file-directory? p)) + (let ([n (string-length p)] + [sn (string-length suffix)]) + (and (>= n sn) + (string=? + suffix + (substring p (- n sn) n)))))) + entries)] + [subdirs (filter file-directory? entries)]) + (loop (append subdirs (cdr dirs)) (append files acc))))) + '())) + +(define (sls->so sls-path) + (string-append (substring sls-path 0 (- (string-length sls-path) 4)) ".so")) + +(define (clean-so! path) + (when (file-exists? path) (delete-file path))) + +(define (fail msg) + (printf "~n FAIL: ~a~n" msg) + (printf "~n=== linux-check FAILED ===~n") + (exit 1)) + +;; ---------- 1. Discover expected jcode modules ---------- + +(printf "[1/4] Discovering jcode modules under lib/jcode...~n") + +(define expected-jcode-sls + (find-files-with-suffix "lib/jcode" ".sls")) + +(printf " found ~a .sls file(s)~n" (length expected-jcode-sls)) +(when (null? expected-jcode-sls) + (fail "no .sls files under lib/jcode — run 'make gen' first")) + +;; ---------- 2. Sanity check: lib/jcode in sync with src/jcode ---------- + +(printf "[2/4] Cross-checking src/jcode → lib/jcode...~n") + +(define expected-src-ss + (find-files-with-suffix "src/jcode" ".ss")) +(printf " found ~a .ss source file(s)~n" (length expected-src-ss)) + +(unless (= (length expected-src-ss) (length expected-jcode-sls)) + (printf " WARNING: src has ~a .ss but lib has ~a .sls — 'make gen' may be stale~n" + (length expected-src-ss) (length expected-jcode-sls))) + +;; ---------- 3. Compile main-binary.ss ---------- + +(printf "[3/4] Compiling main-binary.ss (compile-imported-libraries #t)...~n") + +(clean-so! "main-binary.so") +(clean-so! "main-binary.wpo") + +(guard (exn [#t + (printf "~n FAIL: compile-program raised an exception:~n") + (printf " ~a~n" (condition-message exn)) + (printf "~n=== linux-check FAILED ===~n") + (exit 1)]) + (parameterize ([compile-imported-libraries #t] + [optimize-level 2] + [generate-inspector-information #f]) + (compile-program "main-binary.ss"))) + +(unless (file-exists? "main-binary.so") + (fail "compile-program returned normally but main-binary.so was not produced")) + +(printf " main-binary.so = ~a bytes~n" + (file-length (open-file-input-port "main-binary.so"))) + +;; ---------- 4. Verify all jcode .so files produced ---------- + +(printf "[4/4] Verifying jcode module .so files...~n") + +(define missing-so + (filter (lambda (p) (not (file-exists? p))) + (map sls->so expected-jcode-sls))) + +(cond + [(null? missing-so) + (printf " all ~a jcode .so file(s) produced~n" (length expected-jcode-sls))] + [else + (printf "~n FAIL: ~a jcode .so file(s) missing after compile-program:~n" + (length missing-so)) + (for-each (lambda (p) (printf " - ~a~n" p)) missing-so) + (printf "~n This usually means compile-imported-libraries did not reach~n") + (printf " that module — main-binary.ss may have a missing transitive import.~n") + (printf "~n=== linux-check FAILED ===~n") + (exit 1)]) + +;; ---------- cleanup ---------- + +(clean-so! "main-binary.so") +(clean-so! "main-binary.wpo") + +(printf "~n=== linux-check PASSED ===~n") +(printf "main-binary.ss compiles cleanly — musl build should get past step 1.~n~n") +(exit 0) --- a/src/jcode/provider/provider.ss +++ b/src/jcode/provider/provider.ss @@ -8,7 +8,7 @@ provider-model) (import :std/text/json - (except (std net request) http-post-stream) + :std/net/request :std/net/tls-rustls :std/net/tcp :std/misc/string