Add static binary build system (macOS, Linux, FreeBSD)
ober
1a6fbaeb7c762d63e4f68d9653c3bf18d77d0787
--- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SHIM_DIR := $(CURDIR)/vendor/chez-sqlite NATIVE_LIB_DIR := $(JERBOA_HOME)/lib LDPATH := $(SHIM_DIR):$(SQLITE_LIB_DIR):$(NATIVE_LIB_DIR) -.PHONY: all build gen run test clean repl +.PHONY: all build gen run test clean repl binary install all: build @@ -31,6 +31,34 @@ test: build DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ $(SCHEME) $(LIBDIRS) --script test/run.ss +# --- Static binary targets --- + +binary: build + JERBOA_HOME=$(JERBOA_HOME) \ + DYLD_LIBRARY_PATH=$(LDPATH) LD_LIBRARY_PATH=$(LDPATH) \ + $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src \ + --script build-binary.ss + +install: binary + mkdir -p $(HOME)/.local/bin + cp jcode $(HOME)/.local/bin/jcode + @echo "Installed to ~/.local/bin/jcode" + +# Cross-platform binary targets (require native host or Docker) +# Linux: build on Linux host with Chez installed +# FreeBSD: build on FreeBSD host with Chez installed +# Both use the same build-binary.ss which auto-detects the platform + +linux: build + JERBOA_HOME=$(JERBOA_HOME) \ + $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src \ + --script build-binary.ss + +freebsd: build + JERBOA_HOME=$(JERBOA_HOME) \ + $(SCHEME) -q --libdirs $(JERBOA_HOME)/lib:./lib:vendor/chez-sqlite/src \ + --script build-binary.ss + clean: find lib/jcode -name "*.sls" -delete 2>/dev/null; true find . -name "*.so" -delete new file mode 100644 --- /dev/null +++ b/build-binary.ss @@ -0,0 +1,420 @@ +#!chezscheme +;; Build a native jcode binary. +;; +;; Usage: cd jerboa-code && make binary +;; +;; Produces: ./jcode (single binary with embedded boot files + program) +;; +;; All boot files are embedded as C byte arrays via Sregister_boot_file_bytes. +;; SQLite shim is statically linked; chez-sqlite uses dlopen(NULL) to resolve. + +(import (chezscheme)) + +;; --- Helper: generate C header from binary file --- +(define (file->c-header input-path output-path array-name size-name) + (let* ((port (open-file-input-port input-path)) + (data (get-bytevector-all port)) + (size (bytevector-length data))) + (close-port port) + (call-with-output-file output-path + (lambda (out) + (fprintf out "/* Auto-generated — do not edit */~n") + (fprintf out "static const unsigned char ~a[] = {~n" array-name) + (let loop ((i 0)) + (when (< i size) + (when (= 0 (modulo i 16)) (fprintf out " ")) + (fprintf out "0x~2,'0x" (bytevector-u8-ref data i)) + (when (< (+ i 1) size) (fprintf out ",")) + (when (= 15 (modulo i 16)) (fprintf out "~n")) + (loop (+ i 1)))) + (fprintf out "~n};~n") + (fprintf out "static const unsigned int ~a = ~a;~n" size-name size)) + 'replace) + (printf " ~a: ~a bytes~n" output-path size))) + +;; --- Detect OS --- +(define freebsd? + (let ((mt (symbol->string (machine-type)))) + (or (string=? mt "ta6fb") (string=? mt "a6fb") + (string=? mt "tarm64fb") (string=? mt "arm64fb")))) + +(define (string-has-suffix? s suffix) + (let ((slen (string-length s)) + (suflen (string-length suffix))) + (and (>= slen suflen) + (string=? suffix (substring s (- slen suflen) slen))))) + +(define macos? + (string-has-suffix? (symbol->string (machine-type)) "osx")) + +(define termux? + (and (getenv "PREFIX") + (let ((prefix (getenv "PREFIX"))) + (and (string? prefix) (> (string-length prefix) 0) + (file-exists? (format "~a/bin/termux-info" prefix)))))) + +;; --- Locate Chez install directory --- +(define (find-csv-dir lib-dir mt) + (let ((csv-dir + (let lp ((dirs (guard (e (#t '())) (directory-list lib-dir)))) + (cond + ((null? dirs) #f) + ((and (> (string-length (car dirs)) 3) + (string=? "csv" (substring (car dirs) 0 3))) + (format "~a/~a/~a" lib-dir (car dirs) mt)) + (else (lp (cdr dirs))))))) + (and csv-dir + (file-exists? (format "~a/main.o" csv-dir)) + csv-dir))) + +(define chez-dir + (or (getenv "CHEZ_DIR") + (let ((mt (symbol->string (machine-type))) + (home (getenv "HOME"))) + (or (find-csv-dir (format "~a/.local/lib" home) mt) + (find-csv-dir "/usr/local/lib" mt) + (find-csv-dir "/usr/lib" mt) + (find-csv-dir "/opt/homebrew/lib" mt) + (let ((prefix (getenv "PREFIX"))) + (and prefix (find-csv-dir (format "~a/lib" prefix) mt))))))) + +(unless chez-dir + (display "Error: Cannot find Chez install dir. Set CHEZ_DIR.\n") + (exit 1)) + +(define home (getenv "HOME")) +(define jerboa-dir + (or (getenv "JERBOA_HOME") + (format "~a/mine/jerboa" home))) + +(printf "Chez dir: ~a~n" chez-dir) +(printf "Jerboa dir: ~a~n" jerboa-dir) + +;; Add library search paths +(library-directories + (append + (list (cons (format "~a/lib" jerboa-dir) + (format "~a/lib" jerboa-dir))) + (list (cons "lib" "lib")) + (list (cons "vendor/chez-sqlite/src" "vendor/chez-sqlite/src")) + (list (cons "." ".")) + (library-directories))) + +(define jcode-modules + '("lib/jcode/core/config" + "lib/jcode/core/log" + "lib/jcode/core/session" + "lib/jcode/core/message" + "lib/jcode/core/agent" + "lib/jcode/core/plugin" + "lib/jcode/provider/provider" + "lib/jcode/tool/registry" + "lib/jcode/tool/file" + "lib/jcode/tool/bash" + "lib/jcode/tool/web" + "lib/jcode/tool/batch" + "lib/jcode/tool/git" + "lib/jcode/tool/lsp" + "lib/jcode/mcp/client" + "lib/jcode/ui/cli")) + +;; --- Step 0: Clean stale .so files so WPO can recompile everything --- +(printf "~n[0/6] Cleaning stale .so/.wpo files...~n") +(for-each (lambda (m) + (let ((so (format "~a.so" m)) + (wpo (format "~a.wpo" m))) + (when (file-exists? so) (delete-file so)) + (when (file-exists? wpo) (delete-file wpo)))) + jcode-modules) +(system (format "find ~a/lib -name '*.wpo' -delete 2>/dev/null" jerboa-dir)) + +;; --- Step 0.5: Patch chez-sqlite for static linking --- +;; In the static binary, the shim is compiled into the binary itself. +;; We patch chez-sqlite to use (load-shared-object "") which is dlopen(NULL), +;; resolving symbols already linked into the process (like jerboa-shell does). +(printf "[0.5/6] Patching chez-sqlite for static linking...~n") +(define chez-sqlite-sls "vendor/chez-sqlite/src/chez-sqlite.sls") +(define chez-sqlite-backup "vendor/chez-sqlite/src/chez-sqlite.sls.bak") +(system (format "cp ~a ~a" chez-sqlite-sls chez-sqlite-backup)) +;; For static builds: sqlite3 and the shim are linked into the binary, +;; so load-shared-object is unnecessary. Replace _l1/_l2 with no-ops. +(define (string-replace-first s old new) + (let ((slen (string-length s)) + (olen (string-length old))) + (let lp ((i 0)) + (cond + ((> (+ i olen) slen) s) + ((string=? old (substring s i (+ i olen))) + (string-append (substring s 0 i) new (substring s (+ i olen) slen))) + (else (lp (+ i 1))))))) +(let* ((src (call-with-input-file chez-sqlite-sls get-string-all)) + (src (string-replace-first src + (string-append + "(define _l1\n" + " (or (guard (e [#t #f]) (load-shared-object \"libsqlite3.so\"))\n" + " (guard (e [#t #f]) (load-shared-object \"libsqlite3.dylib\"))\n" + " (error 'chez-sqlite \"Cannot find libsqlite3 (.so or .dylib)\")))") + "(define _l1 (void))")) + (src (string-replace-first src + (string-append + "(define _l2\n" + " (or (guard (e [#t #f]) (load-shared-object \"chez_sqlite_shim.so\"))\n" + " (error 'chez-sqlite \"Cannot find chez_sqlite_shim.so " + (string #\x2014) ;; em dash + " add vendor/chez-sqlite to DYLD_LIBRARY_PATH / LD_LIBRARY_PATH\")))") + "(define _l2 (void))"))) + (call-with-output-file chez-sqlite-sls (lambda (p) (put-string p src)) 'replace)) +(when (file-exists? "vendor/chez-sqlite/src/chez-sqlite.so") + (delete-file "vendor/chez-sqlite/src/chez-sqlite.so")) + +;; --- Step 1: Compile all modules + entry point --- +(printf "[1/6] Compiling all modules (optimize-level 3, WPO)...~n") +(parameterize ((compile-imported-libraries #t) + (optimize-level 3) + (cp0-effort-limit 500) + (cp0-score-limit 50) + (cp0-outer-unroll-limit 1) + (commonization-level 4) + (enable-unsafe-application #t) + (enable-unsafe-variable-reference #t) + (enable-arithmetic-left-associative #t) + (debug-level 0) + (generate-inspector-information #f) + (generate-wpo-files #t)) + (compile-program "main-binary.ss")) + +;; --- Step 2: Whole-program optimization --- +(printf "[2/6] Running whole-program optimization...~n") +(let ((missing (compile-whole-program "main-binary.wpo" "jcode-all.so"))) + (unless (null? missing) + (printf " WPO: ~a libraries not incorporated (missing .wpo):~n" (length missing)) + (for-each (lambda (lib) (printf " ~a~n" lib)) missing))) + +;; --- Step 3: Create boot files + C headers --- +(printf "[3/6] Creating boot file and C headers...~n") + +(define (existing-so-files paths) + (filter file-exists? paths)) + +;; External jerboa/std libraries not incorporated by WPO +;; (dependency order: core first, then modules) +(define jerboa-lib-dir (format "~a/lib" jerboa-dir)) + +(define external-libs + (map (lambda (m) (format "~a/~a.so" jerboa-lib-dir m)) + '("jerboa/core" + "jerboa/runtime" + "std/error" + "std/format" + "std/sort" + "std/pregexp" + "std/sugar" + "std/typed" + "std/misc/string" + "std/misc/list" + "std/misc/alist" + "std/misc/thread" + "std/misc/ports" + "std/misc/retry" + "std/misc/uuid" + "std/os/path" + "std/os/shell" + "std/text/json" + "std/text/glob" + "std/net/tcp" + "std/net/tls-rustls" + "std/net/request" + "std/db/sqlite"))) + +(apply make-boot-file "jcode.boot" '("scheme" "petite") + (append + (existing-so-files external-libs) + (list "vendor/chez-sqlite/src/chez-sqlite.so") + (existing-so-files + (map (lambda (m) (format "~a.so" m)) jcode-modules)))) + +;; Generate C headers for embedding +(file->c-header "jcode-all.so" "jcode_program.h" + "jcode_program_data" "jcode_program_size") +(file->c-header (format "~a/petite.boot" chez-dir) "jcode_petite_boot.h" + "petite_boot_data" "petite_boot_size") +(file->c-header (format "~a/scheme.boot" chez-dir) "jcode_scheme_boot.h" + "scheme_boot_data" "scheme_boot_size") +(file->c-header "jcode.boot" "jcode_boot.h" + "jcode_boot_data" "jcode_boot_size") + +;; --- Step 4: Compile C main and link --- +(printf "[4/6] Compiling and linking...~n") + +;; All chez_sqlite_* symbols used by (chez-sqlite) foreign-procedure calls +(define ffi-symbols + '(;; chez-sqlite shim + "chez_sqlite_open" "chez_sqlite_close" "chez_sqlite_exec" + "chez_sqlite_prepare" "chez_sqlite_finalize" "chez_sqlite_reset" + "chez_sqlite_clear_bindings" "chez_sqlite_step" + "chez_sqlite_column_count" "chez_sqlite_column_name" + "chez_sqlite_column_type" "chez_sqlite_column_int64" + "chez_sqlite_column_double" "chez_sqlite_column_text" + "chez_sqlite_column_bytes" "chez_sqlite_column_blob" + "chez_sqlite_bind_int64" "chez_sqlite_bind_double" + "chez_sqlite_bind_text" "chez_sqlite_bind_blob" + "chez_sqlite_bind_null" "chez_sqlite_last_insert_rowid" + "chez_sqlite_changes" "chez_sqlite_errmsg" + "chez_SQLITE_ROW" "chez_SQLITE_DONE" "chez_SQLITE_OK" + ;; jerboa-native (TLS/rustls) + "jerboa_tls_server_new" "jerboa_tls_server_new_mtls" + "jerboa_tls_server_free" "jerboa_tls_accept" + "jerboa_tls_connect" "jerboa_tls_connect_pinned" + "jerboa_tls_connect_mtls" "jerboa_tls_close" + "jerboa_tls_read" "jerboa_tls_write" "jerboa_tls_flush" + "jerboa_tls_get_fd" "jerboa_tls_set_nonblock" + "jerboa_last_error")) + +(call-with-output-file "jcode-main.c" + (lambda (out) + (fprintf out "/* Auto-generated — do not edit */\n") + (fprintf out "#define _GNU_SOURCE\n") + (fprintf out "#include <stdlib.h>\n") + (fprintf out "#include <stdio.h>\n") + (fprintf out "#include <string.h>\n") + (fprintf out "#include <unistd.h>\n") + (fprintf out "#include \"scheme.h\"\n") + (fprintf out "#include \"jcode_petite_boot.h\"\n") + (fprintf out "#include \"jcode_scheme_boot.h\"\n") + (fprintf out "#include \"jcode_boot.h\"\n") + (fprintf out "#include \"jcode_program.h\"\n") + (fprintf out "\n") + ;; Extern declarations for sqlite shim functions + (for-each (lambda (sym) (fprintf out "extern void ~a();\n" sym)) + ffi-symbols) + (fprintf out "\n") + ;; Callback to register FFI symbols during heap build + (fprintf out "static void register_ffi(void) {\n") + (for-each (lambda (sym) + (fprintf out " Sforeign_symbol(\"~a\", (void*)~a);\n" sym sym)) + ffi-symbols) + (fprintf out "}\n\n") + (fprintf out "int main(int argc, char *argv[]) {\n") + (fprintf out " char prog_path[256];\n") + (fprintf out " const char *tmpdir = getenv(\"TMPDIR\");\n") + (fprintf out " if (!tmpdir) tmpdir = \"/tmp\";\n") + (display " snprintf(prog_path, sizeof(prog_path), \"%s/jcode-XXXXXX\", tmpdir);\n" out) + (fprintf out " int fd = mkstemp(prog_path);\n") + (fprintf out " if (fd < 0) { perror(\"mkstemp\"); return 1; }\n") + (fprintf out " if (write(fd, jcode_program_data, jcode_program_size)\n") + (fprintf out " != (ssize_t)jcode_program_size) {\n") + (fprintf out " perror(\"write\"); close(fd); unlink(prog_path); return 1;\n") + (fprintf out " }\n") + (fprintf out " close(fd);\n") + (fprintf out "\n") + (fprintf out " Sscheme_init(NULL);\n") + (fprintf out " Sregister_boot_file_bytes(\"petite\", (void*)petite_boot_data, petite_boot_size);\n") + (fprintf out " Sregister_boot_file_bytes(\"scheme\", (void*)scheme_boot_data, scheme_boot_size);\n") + (fprintf out " Sregister_boot_file_bytes(\"jcode\", (void*)jcode_boot_data, jcode_boot_size);\n") + (fprintf out " Sbuild_heap(NULL, register_ffi);\n") + (fprintf out " int status = Sscheme_script(prog_path, argc, (const char **)argv);\n") + (fprintf out " unlink(prog_path);\n") + (fprintf out " Sscheme_deinit();\n") + (fprintf out " return status;\n") + (fprintf out "}\n")) + 'replace) + +;; Locate sqlite include path +(define sqlite-inc + (let ((brew-prefix (guard (e (#t #f)) + (let-values (((p o e pid) (open-process-ports + "brew --prefix sqlite 2>/dev/null" + 'block (make-transcoder (utf-8-codec))))) + (let ((line (get-line o))) + (close-port p) (close-port o) (close-port e) + (and (string? line) (not (string=? line "")) + (format "-I~a/include" line))))))) + (or brew-prefix ""))) + +(define sqlite-lib + (let ((brew-prefix (guard (e (#t #f)) + (let-values (((p o e pid) (open-process-ports + "brew --prefix sqlite 2>/dev/null" + 'block (make-transcoder (utf-8-codec))))) + (let ((line (get-line o))) + (close-port p) (close-port o) (close-port e) + (and (string? line) (not (string=? line "")) + (format "-L~a/lib" line))))))) + (or brew-prefix ""))) + +;; Platform-specific link flags +(define link-libs + (cond + (freebsd? + "-lkernel -llz4 -lz -lm -lpthread -lncurses -lsqlite3") + (macos? + (format "-lkernel -llz4 -lz -lm -lpthread -lncurses -liconv ~a -lsqlite3" sqlite-lib)) + (termux? + "-lkernel -llz4 -lz -lm -ldl -lpthread -lncurses -liconv -lsqlite3") + (else + "-lkernel -llz4 -lz -lm -ldl -lpthread -luuid -lncurses -lsqlite3"))) + +;; Compile sqlite shim +(define cc (or (getenv "CC") "cc")) +(let ((rc (system (format "~a -c -fPIC -O2 ~a -o chez-sqlite-shim.o vendor/chez-sqlite/chez_sqlite_shim.c" + cc sqlite-inc)))) + (unless (= rc 0) + (printf "Error: SQLite shim compilation failed~n") + (exit 1))) + +;; Compile main +(let ((rc (system (format "~a -c -I~a -o jcode-main.o jcode-main.c" cc chez-dir)))) + (unless (= rc 0) + (printf "Error: C compilation failed~n") + (exit 1))) + +;; Locate jerboa-native static lib (TLS/rustls) +(define native-lib + (format "~a/jerboa-native-rs/target/release/libjerboa_native.a" jerboa-dir)) +(unless (file-exists? native-lib) + (printf "Warning: ~a not found — TLS will be unavailable~n" native-lib)) + +;; Link — Sforeign_symbol registers FFI symbols +(let ((rc (system (format "~a -o jcode jcode-main.o chez-sqlite-shim.o ~a -L~a ~a -framework Security -framework CoreFoundation" + cc + (if (file-exists? native-lib) native-lib "") + chez-dir link-libs)))) + (unless (= rc 0) + (printf "Error: linking failed~n") + (exit 1))) + +;; --- Step 5: Restore chez-sqlite --- +(printf "[5/6] Restoring chez-sqlite...~n") +(when (file-exists? chez-sqlite-backup) + (system (format "mv ~a ~a" chez-sqlite-backup chez-sqlite-sls))) + +;; --- Step 6: Cleanup --- +(printf "[6/6] Cleaning up intermediate files...~n") +(for-each (lambda (f) + (when (file-exists? f) (delete-file f))) + '("jcode-main.c" + "jcode-main.o" + "chez-sqlite-shim.o" + "jcode_program.h" + "jcode_petite_boot.h" + "jcode_scheme_boot.h" + "jcode_boot.h" + "jcode-all.so" + "jcode.boot" + "main-binary.wpo" + "main-binary.so")) + +;; Clean up .so and .wpo from compile-imported-libraries +(for-each (lambda (m) + (let ((so (format "~a.so" m)) + (wpo (format "~a.wpo" m))) + (when (file-exists? so) (delete-file so)) + (when (file-exists? wpo) (delete-file wpo)))) + jcode-modules) + +(printf "~nDone! Binary: ./jcode~n") +(printf " Install: cp jcode ~a/.local/bin/jcode~n" home) +(let* ((port (open-file-input-port "jcode")) + (size (/ (port-length port) 1048576.0))) + (close-port port) + (printf " Size: ~a MB~n" (/ (round (* size 10)) 10.0))) new file mode 100644 --- /dev/null +++ b/main-binary.ss @@ -0,0 +1,10 @@ +#!chezscheme +;;; jcode binary entry point +;;; In binary mode, all libraries are compiled in via boot files. + +(import (chezscheme) + (jcode core config) + (jcode core agent) + (jcode ui cli)) + +(cli-main (command-line-arguments)) --- a/vendor/chez-sqlite +++ b/vendor/chez-sqlite @@ -1 +1 @@ -Subproject commit 2f5942985ab0b1783b802a4216ec846b8e7afe1f +Subproject commit 11c1031e66ebe961799d8f0862ee8398eae5bd26