build: build jawk with installed jerbuild (no jerboa checkout)
ober
51ae54d9278453ec93b2ef2fe2f0a402350a1080
--- a/.jerbuild +++ b/.jerbuild @@ -1,7 +1,9 @@ -;; Build jawk with a standalone jerbuild binary. +;; Build jawk with a standalone jerbuild binary: `jerbuild build`. +;; +;; jerbuild bundles Chez Scheme + the jerboa stdlib, so NO jerboa source +;; checkout and NO separately-built Chez is required — only a C compiler (cc) +;; for the final link. Pure Jerboa: no jerboa-native-rs and no DuckDB. (entry "bin/jawk-program.ss") (output "jawk") -(requires "cc") -(notes "Pure Jerboa build path. The legacy cross scripts link jerboa-native-rs with tls,sqlite,crypto for static runtime coverage, but DuckDB is not required.") (libdirs "lib") --- a/Makefile +++ b/Makefile @@ -1,92 +1,39 @@ -JERBOA_HOME ?= $(HOME)/mine/jerboa -SCHEME ?= $(JERBOA_HOME)/.chez/bin/scheme -LIBDIRS := lib:$(JERBOA_HOME)/lib -JAWK := $(SCHEME) --libdirs "$(LIBDIRS)" --script bin/jawk.ss +# jerbuild bundles Chez Scheme + the jerboa stdlib under ~/.cache/jerbuild/, +# so building jawk needs only `jerbuild` + a C compiler — no jerboa source +# checkout and no separately-built Chez. +JERBUILD ?= jerbuild +JH := $(shell $(JERBUILD) --jerboa-home 2>/dev/null) +ifeq ($(JH),) +$(error jerbuild not found on PATH (or '$(JERBUILD) --jerboa-home' failed). Install jerbuild, or set JERBUILD=/path/to/jerbuild) +endif -# Cross-compile libdirs — include the cross-built jerboa stdlib so xpatch can -# find (jerboa build) etc. during the ta6le/ta6fb emit pass. -XC_LIBDIRS := lib:$(JERBOA_HOME)/lib:$(JERBOA_HOME)/.chez-cross-ta6le/lib:$(JERBOA_HOME)/.chez-cross-ta6fb/lib +LIBDIRS := --libdirs lib:$(JH)/lib +JAWK_BIN := jawk +.PHONY: all build binary test clean install -.PHONY: all build clean test binary binary-static binary-release linux linux-amd64 freebsd freebsd-amd64 jawk-linux-amd64 jawk-freebsd-amd64 +all: binary -all: build +# Standalone native binary via .jerbuild (entry bin/jawk-program.ss -> jawk). +binary: + $(JERBUILD) build -build: - @echo "Compiling libraries..." - @$(SCHEME) --libdirs "$(LIBDIRS)" --script build.ss - @echo "Done." - -binary: build - @echo "Building jawk binary..." - @$(SCHEME) --libdirs "$(LIBDIRS)" --script build-binary.ss - -binary-release: build - @echo "Building jawk release binary..." - @$(SCHEME) --libdirs "$(LIBDIRS)" --script build-binary.ss --release - -binary-static: build - @echo "Building jawk static binary..." - @$(SCHEME) --libdirs "$(LIBDIRS)" --script build-binary.ss --static - -# Cross-compile: macOS host -> Linux x86_64 musl (static ELF) -linux-amd64: - @command -v x86_64-linux-musl-gcc >/dev/null 2>&1 || { \ - echo "ERROR: x86_64-linux-musl-gcc not found on PATH."; \ - echo "Install the FiloSottile musl-cross toolchain (e.g. via Homebrew tap)."; \ - exit 1; } - @test -d $(JERBOA_HOME)/.chez-cross-ta6le || { \ - echo "ERROR: $(JERBOA_HOME)/.chez-cross-ta6le not found."; \ - echo "Run 'make binary' in $(JERBOA_HOME) first."; \ - exit 1; } - @echo "==> Cross-building jawk for Linux x86_64 (musl static)..." - JERBOA_HOME=$(JERBOA_HOME) $(SCHEME) -q --libdirs "$(XC_LIBDIRS)" --script build-jawk-cross.ss - -linux: linux-amd64 -jawk-linux-amd64: linux-amd64 - -# Cross-compile: macOS host -> FreeBSD 14 x86_64 (dynamic ELF) -freebsd-amd64: - @command -v x86_64-unknown-freebsd14-clang >/dev/null 2>&1 || { \ - echo "ERROR: x86_64-unknown-freebsd14-clang wrapper not found on PATH."; \ - echo "See $(JERBOA_HOME) docs for setting up the FreeBSD cross toolchain."; \ - exit 1; } - @test -d $(JERBOA_HOME)/.chez-cross-ta6fb || { \ - echo "ERROR: $(JERBOA_HOME)/.chez-cross-ta6fb not found."; \ - echo "Run 'make binary' in $(JERBOA_HOME) first (builds Chez xc-ta6fb)."; \ - exit 1; } - @echo "==> Cross-building jawk for FreeBSD 14 x86_64 (dynamic)..." - JERBOA_HOME=$(JERBOA_HOME) $(SCHEME) -q --libdirs "$(XC_LIBDIRS)" --script build-jawk-freebsd-cross.ss - -freebsd: freebsd-amd64 -jawk-freebsd-amd64: freebsd-amd64 +build: binary clean: - find lib -name "*.so" -delete - rm -f jawk jawk-linux-amd64 jawk-freebsd-amd64 - rm -f jawk-linux-amd64-main.c jawk-freebsd-amd64-main.c - rm -f jawk-linux-amd64.wp.so jawk-freebsd-amd64.wp.so - rm -f petite_boot.h scheme_boot.h jawk_program.h - -test: - @echo "--- Test 1: Hello World ---" - @echo "hello world" | $(JAWK) '{print $$1}' - @echo "--- Test 2: Field separator ---" - @echo "a,b,c" | $(JAWK) -F, '{print $$2}' - @echo "--- Test 3: BEGIN/END ---" - @echo "" | $(JAWK) 'BEGIN{print "start"} END{print "end"}' - @echo "--- Test 4: NR counter ---" - @printf "a\nb\nc\n" | $(JAWK) '{print NR, $$0}' - @echo "--- Test 5: Arithmetic ---" - @printf "10\n20\n30\n" | $(JAWK) '{sum += $$1} END {print sum}' - @echo "--- Test 6: Regex ---" - @printf "hello\nworld\nhelium\n" | $(JAWK) '/hel/{print}' - @echo "--- Test 7: User function ---" - @echo "5" | $(JAWK) 'function double(x) { return x * 2 } { print double($$1) }' - @echo "--- Test 8: Associative arrays ---" - @printf "apple\nbanana\napple\ncherry\napple\n" | $(JAWK) '{count[$$1]++} END {for (k in count) print k, count[k]}' - @echo "--- Test 9: String functions ---" - @echo "Hello World" | $(JAWK) '{print length($$0), tolower($$0), substr($$0, 7)}' - @echo "--- Test 10: gsub ---" - @echo "foo bar foo" | $(JAWK) '{gsub(/foo/, "FOO"); print}' - @echo "--- All tests passed ---" + find lib \( -name '*.so' -o -name '*.wpo' \) -delete 2>/dev/null || true + rm -f $(JAWK_BIN) + +install: binary + install -d $(HOME)/.local/bin + install -m 0755 $(JAWK_BIN) $(HOME)/.local/bin/$(JAWK_BIN) + +test: binary + @echo "--- print field ---"; echo "hello world" | ./$(JAWK_BIN) '{print $$1}' + @echo "--- -F, ---"; echo "a,b,c" | ./$(JAWK_BIN) -F, '{print $$2}' + @echo "--- BEGIN/END ---"; echo "" | ./$(JAWK_BIN) 'BEGIN{print "start"} END{print "end"}' + @echo "--- NR ---"; printf "a\nb\nc\n" | ./$(JAWK_BIN) '{print NR, $$0}' + @echo "--- sum ---"; printf "10\n20\n30\n" | ./$(JAWK_BIN) '{sum += $$1} END {print sum}' + @echo "--- assoc arrays ---"; printf "apple\nbanana\napple\n" | ./$(JAWK_BIN) '{c[$$1]++} END {for (k in c) print k, c[k]}' + @echo "--- gsub ---"; echo "foo bar foo" | ./$(JAWK_BIN) '{gsub(/foo/, "FOO"); print}' + @echo "All tests passed." --- a/bin/jawk-program.ss +++ b/bin/jawk-program.ss @@ -1,10 +1,12 @@ -;;;; jawk — AWK interpreter entry point (for binary compilation) -;;;; Uses scheme-start so Sscheme_start() in C passes argv through. +#!chezscheme +;;;; jawk — AWK interpreter entry point (for binary compilation). +;;;; +;;;; jerbuild's main.c runs this file as a program: the top-level forms +;;;; execute and (command-line-arguments) holds argv[1..]. (No scheme-start +;;;; hook fires under jerbuild, so call main directly.) (import (chezscheme) (jerboa-awk main)) (suppress-greeting #t) -(scheme-start - (lambda fns - (main fns))) +(main (command-line-arguments)) deleted file mode 100755 --- a/build-binary.ss +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env scheme --script -;;;; Build a self-contained jawk binary -;;;; -;;;; Strategy: compile-program with compile-imported-libraries creates all -;;;; .so files. We collect them all and pack into the boot file so the -;;;; binary is fully self-contained. - -(import (chezscheme)) - -;; Setup library paths -(let ((jerboa-home (or (getenv "JERBOA_HOME") - (string-append (getenv "HOME") "/mine/jerboa")))) - (library-directories (cons* (cons "lib" "lib") - (cons (string-append jerboa-home "/lib") - (string-append jerboa-home "/lib")) - (library-directories)))) - -;; Find Chez install directory containing scheme.h + boot files. -;; Search order: $JERBOA_HOME/.chez/lib/csv*/<mt> first (self-built), -;; then /usr/lib/csv*/<mt> (system-installed Homebrew/apt/etc). -(define chez-dir - (let* ((mt (symbol->string (machine-type))) - (jhome (or (getenv "JERBOA_HOME") - (string-append (getenv "HOME") "/mine/jerboa"))) - (self-lib (string-append jhome "/.chez/lib")) - (search-dirs (filter (lambda (d) (and d (file-directory? d))) - (list self-lib "/usr/lib")))) - (let outer ((roots search-dirs)) - (cond - ((null? roots) - (error 'build "cannot find Chez Scheme install directory")) - (else - (let inner ((dirs (directory-list (car roots)))) - (cond - ((null? dirs) (outer (cdr roots))) - (else - (let ((candidate (format "~a/~a/~a" (car roots) (car dirs) mt))) - (if (and (> (string-length (car dirs)) 3) - (string=? (substring (car dirs) 0 3) "csv") - (file-exists? (string-append candidate "/scheme.h")) - (file-exists? (string-append candidate "/scheme.boot"))) - candidate - (inner (cdr dirs)))))))))))) - -(printf "Using Chez dir: ~a~%" chez-dir) -(putenv "SCHEMEHEAPDIRS" chez-dir) - -;; Step 1: Compile program + all imported libraries -(define build-dir - (format "/tmp/jerboa-build-jawk-~a" - (mod (time-second (current-time)) 100000))) -(system (format "mkdir -p '~a'" build-dir)) - -(define so-path (string-append build-dir "/program.so")) -(printf "[1/5] Compiling program + libraries...~%") -(parameterize ([compile-imported-libraries #t] - [optimize-level 2]) - (compile-program "bin/jawk-program.ss" so-path)) - -;; Step 2: Collect all compiled .so files (libraries) and create boot file -;; The libraries are compiled in-place next to their .sls sources. -;; We need them in the boot file in dependency order. -(printf "[2/5] Creating boot file...~%") - -;; Collect library .so files in dependency order -(define jerboa-home (or (getenv "JERBOA_HOME") - (string-append (getenv "HOME") "/mine/jerboa"))) - -;; Helper to find a .so file for a library -(define (find-lib-so lib-path) - (let ((so (string-append lib-path ".so"))) - (cond - ((file-exists? (string-append "lib/" so)) (string-append "lib/" so)) - ((file-exists? (string-append jerboa-home "/lib/" so)) - (string-append jerboa-home "/lib/" so)) - (else #f)))) - -;; Jerboa stdlib dependencies (order matters!) -(define jerboa-libs - (filter values - (map find-lib-so - '("std/pregexp" - "std/misc/string" - "std/match2" - "jerboa/runtime" - "jerboa-awk/ast" - "jerboa-awk/value" - "jerboa-awk/runtime" - "jerboa-awk/lexer" - "jerboa-awk/parser" - "jerboa-awk/builtins/string" - "jerboa-awk/builtins/math" - "jerboa-awk/builtins/io" - "jerboa-awk/main")))) - -(printf " Libraries to embed: ~a~%" (length jerboa-libs)) -(for-each (lambda (l) (printf " ~a~%" l)) jerboa-libs) - -(define app-boot (string-append build-dir "/app.boot")) -(apply make-boot-file app-boot '("petite" "scheme") - (append jerboa-libs (list so-path))) - -;; Step 3: Generate C -;; We use file->c-array from (jerboa build) but write our own main() -;; that calls Sscheme_start(argc, argv) to set (command-line) properly. -(printf "[3/5] Generating C...~%") -(eval '(import (jerboa build))) - -(define (boot->c-array path name) - (eval `(file->c-array ,path ,name))) - -(define main-c - (string-append - (boot->c-array (string-append chez-dir "/petite.boot") "petite_boot") - "\n" - (boot->c-array (string-append chez-dir "/scheme.boot") "scheme_boot") - "\n" - (boot->c-array app-boot "app_boot") - "\n" - "#include <scheme.h>\n" - "#include <string.h>\n" - "#include <stdlib.h>\n\n" - "int main(int argc, const char *argv[]) {\n" - " Sscheme_init(NULL);\n\n" - " Sregister_boot_file_bytes(\"petite\", petite_boot, petite_boot_len);\n" - " Sregister_boot_file_bytes(\"scheme\", scheme_boot, scheme_boot_len);\n" - " Sregister_boot_file_bytes(\"app\", app_boot, app_boot_len);\n\n" - " Sbuild_heap(argv[0], NULL);\n" - " return Sscheme_start(argc, argv);\n" - "}\n")) - -(define main-path (string-append build-dir "/main.c")) -(call-with-output-file main-path - (lambda (p) (display main-c p)) - 'replace) - -;; Step 4: Link -(define output-name "jawk") -(printf "[4/5] Linking ~a...~%" output-name) -;; Platform-specific extras: macOS Chez kernel calls iconv/setupterm/tputs. -(define platform-libs - (case (machine-type) - [(tarm64osx ta6osx) "-liconv -lncurses"] - [else "-ldl"])) -(let* ((lflags (format "-L'~a' -lkernel -llz4 -lz -lm -lpthread ~a" chez-dir platform-libs)) - (cmd (format "gcc -rdynamic -I'~a' -o '~a' '~a' ~a 2>&1" - chez-dir output-name main-path lflags)) - (rc (system cmd))) - (if (= rc 0) - (printf "[5/5] Built: ~a (~a bytes)~%" output-name - (let ((p (open-file-input-port output-name))) - (let ((sz (port-length p))) - (close-port p) - sz))) - (begin - (printf "Link failed (rc=~a)~%" rc) - (printf "Command: ~a~%" cmd)))) deleted file mode 100644 --- a/build-jawk-cross.ss +++ /dev/null @@ -1,435 +0,0 @@ -#!chezscheme -;;; build-jawk-cross.ss — Cross-compile jawk from macOS arm64 to Linux x86_64 musl -;;; -;;; Usage: -;;; JERBOA_HOME=/Users/user/mine/jerboa scheme --libdirs <libs> \ -;;; --script build-jawk-cross.ss -;;; -;;; Cross-compile analogue of build-binary.ss that runs on a non-Linux host. -;;; Uses: -;;; - $JERBOA_HOME/.chez-cross-ta6le/ — cross-built Chez install -;;; - $JERBOA_HOME/build/chez/xc-ta6le/s/xpatch — host compiler → ta6le emit -;;; - x86_64-linux-musl-gcc — C compile + final static link -;;; -;;; Produces: jawk-linux-amd64 (static Linux x86_64 ELF, no shared library deps) - -(import (chezscheme)) - -;; ── Params ────────────────────────────────────────────────────────────────── -(define jerboa-home - (or (getenv "JERBOA_HOME") "/Users/user/mine/jerboa")) - -(define cross-prefix (format "~a/.chez-cross-ta6le" jerboa-home)) -(define xpatch (format "~a/build/chez/xc-ta6le/s/xpatch" jerboa-home)) -(define cross-cc (or (getenv "CROSS_CC") "x86_64-linux-musl-gcc")) - -(define output "jawk-linux-amd64") -(define entry-script "bin/jawk-program.ss") - -;; Match feature set used by sibling repos so libjerboa_native.a is reusable -;; across jerboa-shell/jerboa-code/jerboa-awk/etc. -(define cargo-features "tls,sqlite,crypto") - -(define jerboa-native-a - (or (getenv "JERBOA_NATIVE_A") - (format "~a/jerboa-native-rs/target/x86_64-unknown-linux-musl/release/libjerboa_native.a" - jerboa-home))) - -(define cross-csv-dir - (let ([lib (format "~a/lib" cross-prefix)]) - (unless (file-directory? lib) - (error 'build-jawk-cross "cross prefix lib dir missing — run 'make binary' in jerboa first" lib)) - (let* ([entries (directory-list lib)] - [csvs (filter (lambda (e) - (and (>= (string-length e) 3) - (string=? (substring e 0 3) "csv"))) - entries)]) - (when (null? csvs) - (error 'build-jawk-cross "no csv* in cross lib" lib)) - (format "~a/~a/ta6le" lib (car csvs))))) - -(define (require-file p) - (unless (file-exists? p) - (error 'build-jawk-cross "missing file" p))) - -(require-file xpatch) -(require-file (format "~a/libkernel.a" cross-csv-dir)) -(require-file (format "~a/scheme.h" cross-csv-dir)) -(require-file (format "~a/petite.boot" cross-csv-dir)) -(require-file (format "~a/scheme.boot" cross-csv-dir)) -(require-file entry-script) - -(printf "==> build-jawk-cross~n") -(printf " JERBOA_HOME: ~a~n" jerboa-home) -(printf " cross csv-dir: ~a~n" cross-csv-dir) -(printf " xpatch: ~a~n" xpatch) -(printf " cross-cc: ~a~n" cross-cc) -(printf " output: ~a~n" output) -(printf " features: ~a~n~n" cargo-features) - -;; ── Step 0: Build/rebuild jerboa-native-rs for x86_64-linux-musl ─────────── -(define native-features-sentinel - (format "~a/jerboa-native-rs/target/x86_64-unknown-linux-musl/release/.built-with-~a" - jerboa-home - (let ([s (string-copy cargo-features)]) - (let loop ([i 0]) - (cond [(= i (string-length s)) s] - [(char=? (string-ref s i) #\,) - (string-set! s i #\-) (loop (+ i 1))] - [else (loop (+ i 1))]))))) - -(define (rs-source-newer-than? a-path) - (let ([src-dir (format "~a/jerboa-native-rs/src" jerboa-home)]) - (and (file-directory? src-dir) - (file-exists? a-path) - (let ([a-mtime (file-modification-time a-path)]) - (let walk ([dirs (list src-dir)]) - (and (pair? dirs) - (let* ([d (car dirs)] - [entries (map (lambda (e) (format "~a/~a" d e)) - (directory-list d))] - [files (filter (lambda (p) (not (file-directory? p))) entries)] - [subs (filter file-directory? entries)]) - (or (ormap (lambda (f) - (and (let ([n (string-length f)]) - (and (> n 3) - (string=? ".rs" (substring f (- n 3) n)))) - (time>? (file-modification-time f) a-mtime))) - files) - (walk (append subs (cdr dirs))))))))))) - -(define (path-dirname s) - (let loop ([i (- (string-length s) 1)]) - (cond [(< i 0) "."] - [(char=? (string-ref s i) #\/) (substring s 0 i)] - [else (loop (- i 1))]))) - -(define (capture-line cmd) - (call-with-values - (lambda () (open-process-ports cmd (buffer-mode block) (native-transcoder))) - (lambda (to-stdin from-stdout from-stderr pid) - (let ([line (get-line from-stdout)]) - (close-port to-stdin) - (close-port from-stdout) - (close-port from-stderr) - (if (or (eof-object? line) (zero? (string-length line))) #f line))))) - -(define rustup-rustc-path (capture-line "rustup which rustc 2>/dev/null")) -(define rustup-bin-dir (and rustup-rustc-path (path-dirname rustup-rustc-path))) - -(define (try-cargo-build cmd) - (printf " $ ~a~n" cmd) - (zero? (system cmd))) - -(define (rebuild-native-lib!) - (let* ([nrs-dir (format "~a/jerboa-native-rs" jerboa-home)] - [cargo-args (format "build --release --no-default-features --features ~a --target x86_64-unknown-linux-musl" - cargo-features)] - [via-rustup - (and rustup-bin-dir - (format "cd '~a' && env PATH='~a':$PATH RUSTC='~a/rustc' '~a/cargo' ~a" - nrs-dir rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] - [via-path (format "cd '~a' && cargo ~a" nrs-dir cargo-args)] - [ok? (or (and via-rustup (try-cargo-build via-rustup)) - (try-cargo-build via-path))]) - (unless ok? - (error 'build-jawk-cross - (string-append - "failed to build jerboa-native-rs for x86_64-unknown-linux-musl. " - "Required: `rustup target add x86_64-unknown-linux-musl`, plus the " - "FiloSottile/musl-cross toolchain on PATH so cargo finds " - "x86_64-linux-musl-gcc as the linker."))) - (unless (file-exists? jerboa-native-a) - (error 'build-jawk-cross "cargo succeeded but .a missing" jerboa-native-a)) - (call-with-output-file native-features-sentinel - (lambda (out) (display cargo-features out) (newline out)) - 'truncate))) - -(cond - [(not (file-exists? jerboa-native-a)) - (printf "==> jerboa-native-rs missing — building (features=~a)~n" cargo-features) - (rebuild-native-lib!)] - [(not (file-exists? native-features-sentinel)) - (printf "==> jerboa-native-rs sentinel missing — rebuilding (features=~a)~n" cargo-features) - (rebuild-native-lib!)] - [(rs-source-newer-than? jerboa-native-a) - (printf "==> jerboa-native-rs .rs source newer than .a — rebuilding~n") - (rebuild-native-lib!)] - [else - (printf "==> jerboa-native-rs up to date (features=~a)~n" cargo-features)]) - -(printf "~n") - -;; ── Step 0.5: Patch load-shared-object in libraries for static linking ───── -(printf "==> [0.5/6] patching load-shared-object in libraries~n") -(define lso-patch-cmd - (string-append - "perl -i -0777 -pe 's/" - "\\(load-shared-object(?=\\s)" - "(?:[^()\"]++|\"(?:\\\\.|[^\"\\\\])*+\"|" - "(?<bal>\\((?:[^()\"]++|\"(?:\\\\.|[^\"\\\\])*+\"|(?&bal))*+\\))" - ")*+\\)/(void)/g'")) - -(define jerboa-lib-dir (format "~a/lib" jerboa-home)) -(system (format "find '~a' -name '*.sls' -exec ~a {} +" jerboa-lib-dir lso-patch-cmd)) -(when (file-directory? "lib") - (system (format "find lib -name '*.sls' -exec ~a {} +" lso-patch-cmd))) - -;; Delete stale .so files so they get recompiled with the patches. -(system (format "find '~a' -name '*.so' -delete 2>/dev/null" jerboa-lib-dir)) -(system "find lib -name '*.so' -delete 2>/dev/null") - -(define (restore-patched-files!) - (printf "~n==> [cleanup] restoring patched .sls files via git~n") - (system (format "cd '~a' && git ls-files -z -- '*.sls' | xargs -0 git checkout -- 2>/dev/null" jerboa-home)) - (system "git ls-files -z -- 'lib/*.sls' | xargs -0 git checkout -- 2>/dev/null")) - -;; ── Stage 1: load xpatch (target=ta6le emit mode) ────────────────────────── -(define orig-libdirs (library-directories)) -(printf "==> [1/6] loading xpatch (compiler -> ta6le emit mode)~n") -(load xpatch) -(library-directories orig-libdirs) - -(compile-imported-libraries #t) -(generate-wpo-files #t) - -;; ── Stage 2: compile-program entry-script ────────────────────────────────── -(printf "==> [2/6] compile-program ~a~n" entry-script) -(guard (e [#t (restore-patched-files!) (raise e)]) - (compile-program entry-script)) - -;; compile-program emits the .wpo next to the source. Strip ".ss" to find it. -(define entry-wpo - (let ([n (string-length entry-script)]) - (string-append (substring entry-script 0 (- n 3)) ".wpo"))) - -;; ── Stage 3: compile-whole-program → wpo .so ─────────────────────────────── -(define wpo-output (string-append output ".wp.so")) -(printf "==> [3/6] compile-whole-program ~a -> ~a~n" entry-wpo wpo-output) -(guard (e [#t (restore-patched-files!) (raise e)]) - (compile-whole-program entry-wpo wpo-output #t)) - -;; ── Stage 4: embed boot files + program as C arrays ──────────────────────── -(define (embed-as-c-array in-path var-name out-path) - (let* ([bv (call-with-port (open-file-input-port in-path) get-bytevector-all)] - [n (bytevector-length bv)]) - (call-with-port (open-file-output-port out-path - (file-options no-fail) - (buffer-mode block) - (native-transcoder)) - (lambda (out) - (display (format "static const unsigned char ~a[] = {\n" var-name) out) - (let loop ([i 0]) - (when (< i n) - (display (format "0x~2,'0x," (bytevector-u8-ref bv i)) out) - (when (= (mod (+ i 1) 16) 0) (newline out)) - (loop (+ i 1)))) - (when (positive? n) (newline out)) - (display "};\n" out) - (display (format "static const unsigned int ~a_size = sizeof(~a);\n" - var-name var-name) - out))) - (printf " embed ~a (~a bytes) -> ~a~n" in-path n out-path))) - -(printf "==> [4/6] embed boot files + program as C arrays~n") -(embed-as-c-array (format "~a/petite.boot" cross-csv-dir) "petite_boot" "petite_boot.h") -(embed-as-c-array (format "~a/scheme.boot" cross-csv-dir) "scheme_boot" "scheme_boot.h") -(embed-as-c-array wpo-output "jawk_program" "jawk_program.h") - -;; ── Stage 5: generate main.c ─────────────────────────────────────────────── -;; jawk has no chez-sqlite, no TUI, no project-local shim — only landlock. -(define ffi-shim-symbols - '(;; landlock-shim (~/mine/jerboa/support/landlock-shim.c) - "jerboa_landlock_sandbox")) - -(define jerboa-native-symbols - '(;; jerboa-native (TLS/rustls) - "jerboa_tls_server_new" "jerboa_tls_server_new_mtls" - "jerboa_tls_server_free" "jerboa_tls_accept" - "jerboa_tls_connect" "jerboa_tls_connect_pinned" - "jerboa_tls_connect_mtls" "jerboa_tls_close" - "jerboa_tls_read" "jerboa_tls_write" "jerboa_tls_flush" - "jerboa_tls_get_fd" "jerboa_tls_set_nonblock" - "jerboa_last_error" - ;; jerboa-native (crypto) - "jerboa_sha1" "jerboa_sha256" "jerboa_sha384" "jerboa_sha512" - "jerboa_random_bytes" "jerboa_timing_safe_equal" - "jerboa_hmac_sha256" "jerboa_hmac_sha256_verify" - "jerboa_aead_seal" "jerboa_aead_open" - "jerboa_chacha20_seal" "jerboa_chacha20_open" - "jerboa_scrypt" - "jerboa_pbkdf2_derive" "jerboa_pbkdf2_verify" - "jerboa_argon2id_hash" "jerboa_argon2id_verify" - ;; jerboa-native (regex) — visit-time even if jawk never matches - "jerboa_regex_compile" "jerboa_regex_is_match" - "jerboa_regex_find" "jerboa_regex_replace_all" "jerboa_regex_free" - ;; jerboa-native (aproc) — (std os aproc) - "jerboa_aproc_spawn" "jerboa_aproc_spawn_pty" - "jerboa_aproc_set_nonblock" "jerboa_aproc_killpg" "jerboa_aproc_wait4" - ;; jerboa-native (sqlite via rusqlite) — kept since .a includes them - "jerboa_sqlite_open" "jerboa_sqlite_close" "jerboa_sqlite_exec" - "jerboa_sqlite_prepare" "jerboa_sqlite_finalize" "jerboa_sqlite_reset" - "jerboa_sqlite_step" "jerboa_sqlite_changes" "jerboa_sqlite_errmsg" - "jerboa_sqlite_last_insert_rowid" - "jerboa_sqlite_bind_int" "jerboa_sqlite_bind_double" - "jerboa_sqlite_bind_text" "jerboa_sqlite_bind_blob" "jerboa_sqlite_bind_null" - "jerboa_sqlite_column_count" "jerboa_sqlite_column_type" - "jerboa_sqlite_column_int" "jerboa_sqlite_column_double" - "jerboa_sqlite_column_text" "jerboa_sqlite_column_blob" - "jerboa_sqlite_column_name" - ;; jerboa-native (net/io) - "jerboa_writev2")) - -(define posix-symbols - '(;; sockets — std/net/tcp, std/net/udp - "socket" "bind" "listen" "accept" "connect" "close" - "setsockopt" "getsockopt" "getsockname" - "htons" "ntohs" "inet_pton" "inet_addr" - "read" "write" "recvfrom" "sendto" - ;; DNS — std/net/resolve - "getaddrinfo" "freeaddrinfo" "inet_ntop" - ;; process/signals — std/misc/process, std/os/signal, aproc - "fork" "waitpid" "kill" "getpid" "getppid" - "getpgid" "setpgid" "setsid" - "getuid" "geteuid" "getgid" "getegid" - "sigemptyset" "sigfillset" "sigaddset" "sigdelset" - "sigismember" "sigprocmask" - ;; terminal/fd - "isatty" "tcgetattr" "tcsetattr" "tcgetpgrp" "tcsetpgrp" - "pipe" "dup" "dup2" "lseek" - ;; env/misc - "setenv" "unsetenv" "strerror" "localtime" "strftime" - "sysconf" "getpagesize" "getrlimit" - "system" "getenv" "putenv" "_exit" "execvp" "execve" - "__errno_location" "fcntl")) - -(define main-c-path (string-append output "-main.c")) - -(define (emit-c out) - (display "/* Generated by build-jawk-cross.ss — do not edit by hand. */\n" out) - (display "#define _GNU_SOURCE\n" out) - (display "#include <stdlib.h>\n" out) - (display "#include <string.h>\n" out) - (display "#include <stdio.h>\n" out) - (display "#include <unistd.h>\n" out) - (display "#include <sys/mman.h>\n" out) - (display "#include <sys/types.h>\n" out) - (display "#include <sys/stat.h>\n" out) - (display "#include <sys/wait.h>\n" out) - (display "#include <sys/resource.h>\n" out) - (display "#include <sys/socket.h>\n" out) - (display "#include <netinet/in.h>\n" out) - (display "#include <arpa/inet.h>\n" out) - (display "#include <netdb.h>\n" out) - (display "#include <termios.h>\n" out) - (display "#include <fcntl.h>\n" out) - (display "#include <signal.h>\n" out) - (display "#include <time.h>\n" out) - (display "#include <errno.h>\n" out) - (display "#include \"scheme.h\"\n" out) - (display "#include \"petite_boot.h\"\n" out) - (display "#include \"scheme_boot.h\"\n" out) - (display "#include \"jawk_program.h\"\n\n" out) - (display "/* dlopen stubs for static linking. */\n" out) - (display "void *dlopen(const char *f, int flags) { (void)flags; return f ? NULL : (void*)1; }\n" out) - (display "void *dlsym(void *h, const char *s) { (void)h; (void)s; return NULL; }\n" out) - (display "int dlclose(void *h) { (void)h; return 0; }\n" out) - (display "static char dlerror_msg[] = \"static binary: dlopen of named libraries is stubbed\";\n" out) - (display "char *dlerror(void) { return dlerror_msg; }\n\n" out) - (display "/* Shim function decls — landlock-shim.c */\n" out) - (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) ffi-shim-symbols) - (display "\n/* libjerboa_native.a — features=tls,sqlite,crypto */\n" out) - (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) jerboa-native-symbols) - (display "\nextern int *__errno_location(void);\n" out) - (newline out) - (display "static void register_ffi_symbols(void) {\n" out) - (for-each (lambda (n) - (fprintf out " Sforeign_symbol(\"~a\", (void*)~a);\n" n n)) - ffi-shim-symbols) - (for-each (lambda (n) - (fprintf out " Sforeign_symbol(\"~a\", (void*)~a);\n" n n)) - jerboa-native-symbols) - (for-each (lambda (n) - (fprintf out " Sforeign_symbol(\"~a\", (void*)~a);\n" n n)) - posix-symbols) - (display "}\n\n" out) - (display "int main(int argc, char *argv[]) {\n" out) - (display " setenv(\"JERBOA_STATIC\", \"1\", 1);\n\n" out) - (display " int fd = memfd_create(\"jawk-program\", MFD_CLOEXEC);\n" out) - (display " char prog_path[64];\n" out) - (display " if (fd >= 0) {\n" out) - (display " if (write(fd, jawk_program, jawk_program_size) != (ssize_t)jawk_program_size) {\n" out) - (display " perror(\"write memfd\"); close(fd); return 1;\n" out) - (display " }\n" out) - (display " snprintf(prog_path, sizeof(prog_path), \"/proc/self/fd/%d\", fd);\n" out) - (display " } else {\n" out) - (display " const char *tmp = getenv(\"TMPDIR\"); if (!tmp) tmp = \"/tmp\";\n" out) - (display " snprintf(prog_path, sizeof(prog_path), \"%s/.jawk-prog-%d.so\", tmp, getpid());\n" out) - (display " FILE *fp = fopen(prog_path, \"wb\");\n" out) - (display " if (!fp) { perror(\"fopen tmpfile\"); return 1; }\n" out) - (display " if (fwrite(jawk_program, 1, jawk_program_size, fp) != jawk_program_size) {\n" out) - (display " perror(\"fwrite\"); fclose(fp); unlink(prog_path); return 1;\n" out) - (display " }\n" out) - (display " fclose(fp);\n" out) - (display " }\n\n" out) - (display " Sscheme_init(NULL);\n" out) - (display " Sregister_boot_file_bytes(\"petite\", (void *)petite_boot, petite_boot_size);\n" out) - (display " Sregister_boot_file_bytes(\"scheme\", (void *)scheme_boot, scheme_boot_size);\n" out) - (display " Sbuild_heap(NULL, NULL);\n" out) - (display " register_ffi_symbols();\n\n" out) - (display " int status = Sscheme_program(prog_path, argc, (const char **)argv);\n\n" out) - (display " if (fd >= 0) close(fd); else unlink(prog_path);\n" out) - (display " Sscheme_deinit();\n" out) - (display " return status;\n" out) - (display "}\n" out)) - -(call-with-port (open-file-output-port main-c-path - (file-options no-fail) (buffer-mode block) (native-transcoder)) - emit-c) -(printf "==> [5/6] generated ~a (~a shim + ~a native + ~a posix)~n" - main-c-path - (length ffi-shim-symbols) - (length jerboa-native-symbols) - (length posix-symbols)) - -;; ── Stage 6: compile + link with cross-cc ────────────────────────────────── -(printf "==> [6/6] compile + link with ~a~n" cross-cc) - -(define landlock-shim-c (format "~a/support/landlock-shim.c" jerboa-home)) -(require-file landlock-shim-c) -(require-file jerboa-native-a) - -(define link-cmd - (format - (string-append - "~a -O2 -static -Wl,--export-dynamic " - "-I~a " ;; scheme.h - "-o ~a " ;; output - "~a ~a " ;; main.c + landlock-shim - "~a/libkernel.a ~a/libz.a ~a/liblz4.a " ;; chez kernel - "~a " ;; libjerboa_native.a - "-Wl,--allow-multiple-definition " - "-Wl,--defsym=_dl_find_object=0 " - "-lm -ldl -lpthread") - cross-cc cross-csv-dir output - main-c-path landlock-shim-c - cross-csv-dir cross-csv-dir cross-csv-dir - jerboa-native-a)) -(printf " ~a~n" link-cmd) -(let ([rc (system link-cmd)]) - (unless (zero? rc) - (restore-patched-files!) - (error 'build-jawk-cross "cross-link failed" rc))) - -;; ── Cleanup: restore patched files ────────────────────────────────────────── -(restore-patched-files!) - -;; Clean up intermediate compile-program outputs. -(for-each (lambda (f) - (when (file-exists? f) (delete-file f))) - (list (string-append (substring entry-script 0 (- (string-length entry-script) 3)) ".so") - entry-wpo)) - -(printf "~n=== Build complete: ~a ===~n" output) -(system (format "ls -lh ~a" output)) -(system (format "file ~a" output)) deleted file mode 100644 --- a/build-jawk-freebsd-cross.ss +++ /dev/null @@ -1,423 +0,0 @@ -#!chezscheme -;;; build-jawk-freebsd-cross.ss — Cross-compile jawk from macOS arm64 to FreeBSD 14 amd64 -;;; -;;; Usage: -;;; JERBOA_HOME=/Users/user/mine/jerboa scheme --libdirs <libs> \ -;;; --script build-jawk-freebsd-cross.ss -;;; -;;; FreeBSD analogue of build-jawk-cross.ss. Uses: -;;; - $JERBOA_HOME/.chez-cross-ta6fb/ — cross-built Chez install (FreeBSD) -;;; - $JERBOA_HOME/build/chez/xc-ta6fb/s/xpatch — host compiler → ta6fb emit -;;; - x86_64-unknown-freebsd14-clang — macOS clang+lld with FreeBSD sysroot -;;; -;;; Produces: jawk-freebsd-amd64 (dynamic FreeBSD x86_64 ELF; depends on -;;; libc.so.7, libm.so, libthr.so, libutil.so on the target host). - -(import (chezscheme)) - -;; ── Params ────────────────────────────────────────────────────────────────── -(define jerboa-home - (or (getenv "JERBOA_HOME") "/Users/user/mine/jerboa")) - -(define cross-prefix (format "~a/.chez-cross-ta6fb" jerboa-home)) -(define xpatch (format "~a/build/chez/xc-ta6fb/s/xpatch" jerboa-home)) -(define cross-cc (or (getenv "CROSS_CC") "x86_64-unknown-freebsd14-clang")) - -(define output "jawk-freebsd-amd64") -(define entry-script "bin/jawk-program.ss") - -(define cargo-features "tls,sqlite,crypto") - -(define jerboa-native-a - (or (getenv "JERBOA_NATIVE_A") - (format "~a/jerboa-native-rs/target/x86_64-unknown-freebsd/release/libjerboa_native.a" - jerboa-home))) - -(define cross-csv-dir - (let ([lib (format "~a/lib" cross-prefix)]) - (unless (file-directory? lib) - (error 'build-jawk-freebsd-cross "cross prefix lib dir missing — run 'make binary' in jerboa first" lib)) - (let* ([entries (directory-list lib)] - [csvs (filter (lambda (e) - (and (>= (string-length e) 3) - (string=? (substring e 0 3) "csv"))) - entries)]) - (when (null? csvs) - (error 'build-jawk-freebsd-cross "no csv* in cross lib" lib)) - (format "~a/~a/ta6fb" lib (car csvs))))) - -(define (require-file p) - (unless (file-exists? p) - (error 'build-jawk-freebsd-cross "missing file" p))) - -(require-file xpatch) -(require-file (format "~a/libkernel.a" cross-csv-dir)) -(require-file (format "~a/scheme.h" cross-csv-dir)) -(require-file (format "~a/petite.boot" cross-csv-dir)) -(require-file (format "~a/scheme.boot" cross-csv-dir)) -(require-file entry-script) - -(printf "==> build-jawk-freebsd-cross~n") -(printf " JERBOA_HOME: ~a~n" jerboa-home) -(printf " cross csv-dir: ~a~n" cross-csv-dir) -(printf " xpatch: ~a~n" xpatch) -(printf " cross-cc: ~a~n" cross-cc) -(printf " output: ~a~n" output) -(printf " features: ~a~n~n" cargo-features) - -;; ── Step 0: Build/rebuild jerboa-native-rs for x86_64-unknown-freebsd ────── -(define native-features-sentinel - (format "~a/jerboa-native-rs/target/x86_64-unknown-freebsd/release/.built-with-~a" - jerboa-home - (let ([s (string-copy cargo-features)]) - (let loop ([i 0]) - (cond [(= i (string-length s)) s] - [(char=? (string-ref s i) #\,) - (string-set! s i #\-) (loop (+ i 1))] - [else (loop (+ i 1))]))))) - -(define (rs-source-newer-than? a-path) - (let ([src-dir (format "~a/jerboa-native-rs/src" jerboa-home)]) - (and (file-directory? src-dir) - (file-exists? a-path) - (let ([a-mtime (file-modification-time a-path)]) - (let walk ([dirs (list src-dir)]) - (and (pair? dirs) - (let* ([d (car dirs)] - [entries (map (lambda (e) (format "~a/~a" d e)) - (directory-list d))] - [files (filter (lambda (p) (not (file-directory? p))) entries)] - [subs (filter file-directory? entries)]) - (or (ormap (lambda (f) - (and (let ([n (string-length f)]) - (and (> n 3) - (string=? ".rs" (substring f (- n 3) n)))) - (time>? (file-modification-time f) a-mtime))) - files) - (walk (append subs (cdr dirs))))))))))) - -(define (path-dirname s) - (let loop ([i (- (string-length s) 1)]) - (cond [(< i 0) "."] - [(char=? (string-ref s i) #\/) (substring s 0 i)] - [else (loop (- i 1))]))) - -(define (capture-line cmd) - (call-with-values - (lambda () (open-process-ports cmd (buffer-mode block) (native-transcoder))) - (lambda (to-stdin from-stdout from-stderr pid) - (let ([line (get-line from-stdout)]) - (close-port to-stdin) - (close-port from-stdout) - (close-port from-stderr) - (if (or (eof-object? line) (zero? (string-length line))) #f line))))) - -(define rustup-rustc-path (capture-line "rustup which rustc 2>/dev/null")) -(define rustup-bin-dir (and rustup-rustc-path (path-dirname rustup-rustc-path))) - -(define (try-cargo-build cmd) - (printf " $ ~a~n" cmd) - (zero? (system cmd))) - -(define (rebuild-native-lib!) - (let* ([nrs-dir (format "~a/jerboa-native-rs" jerboa-home)] - [cargo-args (format "build --release --no-default-features --features ~a --target x86_64-unknown-freebsd" - cargo-features)] - [cc-env (string-append - "CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_LINKER=x86_64-unknown-freebsd14-clang " - "CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd14-clang " - "AR_x86_64_unknown_freebsd=/opt/homebrew/opt/llvm/bin/llvm-ar ")] - [via-rustup - (and rustup-bin-dir - (format "cd '~a' && env ~aPATH='~a':$PATH RUSTC='~a/rustc' '~a/cargo' ~a" - nrs-dir cc-env rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] - [via-path (format "cd '~a' && env ~acargo ~a" nrs-dir cc-env cargo-args)] - [ok? (or (and via-rustup (try-cargo-build via-rustup)) - (try-cargo-build via-path))]) - (unless ok? - (error 'build-jawk-freebsd-cross - (string-append - "failed to build jerboa-native-rs for x86_64-unknown-freebsd. " - "Required: `rustup target add x86_64-unknown-freebsd`, plus the " - "x86_64-unknown-freebsd14-clang wrapper on PATH and " - "/opt/homebrew/opt/llvm/bin/llvm-ar from the LLVM Homebrew formula."))) - (unless (file-exists? jerboa-native-a) - (error 'build-jawk-freebsd-cross "cargo succeeded but .a missing" jerboa-native-a)) - (call-with-output-file native-features-sentinel - (lambda (out) (display cargo-features out) (newline out)) - 'truncate))) - -(cond - [(not (file-exists? jerboa-native-a)) - (printf "==> jerboa-native-rs missing — building (features=~a)~n" cargo-features) - (rebuild-native-lib!)] - [(not (file-exists? native-features-sentinel)) - (printf "==> jerboa-native-rs sentinel missing — rebuilding (features=~a)~n" cargo-features) - (rebuild-native-lib!)] - [(rs-source-newer-than? jerboa-native-a) - (printf "==> jerboa-native-rs .rs source newer than .a — rebuilding~n") - (rebuild-native-lib!)] - [else - (printf "==> jerboa-native-rs up to date (features=~a)~n" cargo-features)]) - -(printf "~n") - -;; ── Step 0.5: Patch load-shared-object in libraries ──────────────────────── -(printf "==> [0.5/6] patching load-shared-object in libraries~n") -(define lso-patch-cmd - (string-append - "perl -i -0777 -pe 's/" - "\\(load-shared-object(?=\\s)" - "(?:[^()\"]++|\"(?:\\\\.|[^\"\\\\])*+\"|" - "(?<bal>\\((?:[^()\"]++|\"(?:\\\\.|[^\"\\\\])*+\"|(?&bal))*+\\))" - ")*+\\)/(void)/g'")) - -(define jerboa-lib-dir (format "~a/lib" jerboa-home)) -(system (format "find '~a' -name '*.sls' -exec ~a {} +" jerboa-lib-dir lso-patch-cmd)) -(when (file-directory? "lib") - (system (format "find lib -name '*.sls' -exec ~a {} +" lso-patch-cmd))) - -(system (format "find '~a' -name '*.so' -delete 2>/dev/null" jerboa-lib-dir)) -(system "find lib -name '*.so' -delete 2>/dev/null") - -(define (restore-patched-files!) - (printf "~n==> [cleanup] restoring patched .sls files via git~n") - (system (format "cd '~a' && git ls-files -z -- '*.sls' | xargs -0 git checkout -- 2>/dev/null" jerboa-home)) - (system "git ls-files -z -- 'lib/*.sls' | xargs -0 git checkout -- 2>/dev/null")) - -;; ── Stage 1: load xpatch (target=ta6fb emit mode) ────────────────────────── -(define orig-libdirs (library-directories)) -(printf "==> [1/6] loading xpatch (compiler -> ta6fb emit mode)~n") -(load xpatch) -(library-directories orig-libdirs) - -(compile-imported-libraries #t) -(generate-wpo-files #t) - -;; ── Stage 2: compile-program entry-script ────────────────────────────────── -(printf "==> [2/6] compile-program ~a~n" entry-script) -(guard (e [#t (restore-patched-files!) (raise e)]) - (compile-program entry-script)) - -(define entry-wpo - (let ([n (string-length entry-script)]) - (string-append (substring entry-script 0 (- n 3)) ".wpo"))) - -;; ── Stage 3: compile-whole-program → wpo .so ─────────────────────────────── -(define wpo-output (string-append output ".wp.so")) -(printf "==> [3/6] compile-whole-program ~a -> ~a~n" entry-wpo wpo-output) -(guard (e [#t (restore-patched-files!) (raise e)]) - (compile-whole-program entry-wpo wpo-output #t)) - -;; ── Stage 4: embed boot files + program as C arrays ──────────────────────── -(define (embed-as-c-array in-path var-name out-path) - (let* ([bv (call-with-port (open-file-input-port in-path) get-bytevector-all)] - [n (bytevector-length bv)]) - (call-with-port (open-file-output-port out-path - (file-options no-fail) - (buffer-mode block) - (native-transcoder)) - (lambda (out) - (display (format "static const unsigned char ~a[] = {\n" var-name) out) - (let loop ([i 0]) - (when (< i n) - (display (format "0x~2,'0x," (bytevector-u8-ref bv i)) out) - (when (= (mod (+ i 1) 16) 0) (newline out)) - (loop (+ i 1)))) - (when (positive? n) (newline out)) - (display "};\n" out) - (display (format "static const unsigned int ~a_size = sizeof(~a);\n" - var-name var-name) - out))) - (printf " embed ~a (~a bytes) -> ~a~n" in-path n out-path))) - -(printf "==> [4/6] embed boot files + program as C arrays~n") -(embed-as-c-array (format "~a/petite.boot" cross-csv-dir) "petite_boot" "petite_boot.h") -(embed-as-c-array (format "~a/scheme.boot" cross-csv-dir) "scheme_boot" "scheme_boot.h") -(embed-as-c-array wpo-output "jawk_program" "jawk_program.h") - -;; ── Stage 5: generate main.c ─────────────────────────────────────────────── -(define ffi-shim-symbols '()) - -;; Linux-only symbols — return-error stubs.