Add `jerbuild binary` subcommand for standalone native builds
ober
436f76158bdd4a334912f2af401c57c9220b9ee9
--- a/jerbuild.ss +++ b/jerbuild.ss @@ -1399,6 +1399,237 @@ (loop (cdr rest) wpo? (cons (car rest) files))])))) ;;;; ============================================================ +;;;; Binary subcommand — build a standalone executable +;;;; ============================================================ + +(define (machine-type->os-libs mt) + ;; Pick OS link flags from the host machine type. Mirrors build-binary.sh's + ;; native-build defaults — jerbuild always builds for the host. + (let ([s (symbol->string mt)]) + (cond + [(string-ends-with? s "osx") "-lm -lpthread -lncurses -liconv"] + [(string-ends-with? s "fb") "-lm -lpthread -lncurses -L/usr/local/lib -liconv"] + [(string-ends-with? s "le") "-lm -ldl -lpthread -lncurses"] + [else "-lm -lpthread -lncurses"]))) + +(define (parse-cc-flag args) + ;; Scan for --cc <prog> (or --cc=<prog>). Returns (values cc remaining). + (let loop ([args args] [cc #f] [acc '()]) + (cond + [(null? args) (values cc (reverse acc))] + [(string=? (car args) "--cc") + (when (null? (cdr args)) + (error 'jerbuild "--cc requires a value")) + (loop (cddr args) (cadr args) acc)] + [(string-starts-with? (car args) "--cc=") + (loop (cdr args) + (substring (car args) 5 (string-length (car args))) + acc)] + [else (loop (cdr args) cc (cons (car args) acc))]))) + +(define (shell-quote s) + ;; Wrap s in single quotes, escaping any embedded single quote. + (let* ([len (string-length s)] + [out (open-output-string)]) + (write-char #\' out) + (let loop ([i 0]) + (when (< i len) + (let ([c (string-ref s i)]) + (if (char=? c #\') + (display "'\\''" out) + (write-char c out)) + (loop (+ i 1))))) + (write-char #\' out) + (get-output-string out))) + +(define (embed-bytes-as-c-array source-path header-path var-name) + ;; Generate header-path containing `static const unsigned char <var>_data[]` + ;; + `static const unsigned int <var>_size`. Offloads byte-stream conversion + ;; to od + sed for speed (same as support/build-binary.sh). + (let ([cmd + (string-append + "{ printf 'static const unsigned char " var-name "_data[] = {\\n'; " + "od -An -tx1 -v " (shell-quote source-path) + " | sed -e '/^[[:space:]]*$/d' " + " -e 's/^ *//;s/ *$//;s/ */ /g;s/ /,0x/g;s/^/0x/;s/$/,/'; " + "printf '};\\nstatic const unsigned int " var-name "_size = sizeof(" + var-name "_data);\\n'; " + "} > " (shell-quote header-path))]) + (unless (zero? (system cmd)) + (error 'jerbuild + (format "embed-bytes: failed for ~a -> ~a" source-path header-path))))) + +(define *binary-main-c-template* + ;; Generated main.c for the standalone binary. Mirrors the structure of + ;; support/build-binary.sh's CMAIN block. + "#include \"scheme.h\" +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/types.h> + +#include \"petite_boot.h\" +#include \"scheme_boot.h\" +#include \"program_boot.h\" + +static const char *write_program_tmpfile(void) { + static char path[] = \"/tmp/jerboa-prog-XXXXXX\"; + int fd = mkstemp(path); + if (fd < 0) { perror(\"mkstemp\"); exit(1); } + ssize_t n = write(fd, program_boot_data, program_boot_size); + if (n != (ssize_t)program_boot_size) { + perror(\"write\"); close(fd); unlink(path); exit(1); + } + close(fd); + return path; +} + +int main(int argc, const char *argv[]) { + Sscheme_init(NULL); + Sregister_boot_file_bytes(\"petite\", + (void *)petite_boot_data, petite_boot_size); + Sregister_boot_file_bytes(\"scheme\", + (void *)scheme_boot_data, scheme_boot_size); + Sbuild_heap(NULL, NULL); + + const char *prog_path = write_program_tmpfile(); + int status = Sscheme_program(prog_path, argc, argv); + unlink(prog_path); + + Sscheme_deinit(); + return status; +} +") + +(define (run-binary args) + ;; jerbuild binary [--libdirs <p>] [--cc <prog>] <entry.ss> <output> + ;; + ;; Builds a standalone executable that bundles Chez + the user's entry script + ;; (WPO compiled). The user gets a single binary with no external deps beyond + ;; system libc + libncurses. + (let-values ([(libs rest1) (parse-libdirs-flag args)]) + (let-values ([(cc-arg rest) (parse-cc-flag rest1)]) + (when (< (length rest) 2) + (error 'jerbuild + "binary: usage: jerbuild binary [--libdirs P] [--cc CC] <entry.ss> <output>")) + (let* ([entry (car rest)] + [output (cadr rest)] + [cc (or cc-arg (or (getenv "CC") "cc"))] + [bundle-dir (or (getenv "JERBUILD_BUNDLE_DIR") + (error 'jerbuild + "binary: JERBUILD_BUNDLE_DIR not set (jerbuild internal bug)"))] + [mt (machine-type)] + [csv-dir (format "~a/csv/~a" bundle-dir mt)] + [petite-boot (format "~a/petite.boot" csv-dir)] + [scheme-boot (format "~a/scheme.boot" csv-dir)] + [libkernel (format "~a/libkernel.a" csv-dir)] + [scheme-h (format "~a/scheme.h" csv-dir)]) + + (unless (file-exists? entry) + (error 'jerbuild (format "binary: entry not found: ~a" entry))) + (unless (and (file-exists? libkernel) (file-exists? scheme-h) + (file-exists? petite-boot) (file-exists? scheme-boot)) + (error 'jerbuild + (format "binary: required Chez files missing under ~a (bundle was built without csv files for this machine-type)" + csv-dir))) + + ;; Libdirs: user-supplied first (so they can shadow), then bundled stdlib. + ;; Dedupe so passing --libdirs $bundle/lib doesn't double-add. + ;; User libs are redirected to obj-dir so compile-imported-libraries + ;; writes .so/.wpo there instead of next to user sources. Bundle lib + ;; keeps its plain mapping because precompiled .so/.wpo are bundled + ;; alongside the .sls sources (see build-jerbuild.sh step 2). + (let* ([bundle-lib (format "~a/lib" bundle-dir)] + [user-libs (let loop ([xs libs] [seen '()] [acc '()]) + (cond + [(null? xs) (reverse acc)] + [(or (member (car xs) seen) + (string=? (car xs) bundle-lib)) + (loop (cdr xs) seen acc)] + [else + (loop (cdr xs) + (cons (car xs) seen) + (cons (car xs) acc))]))]) + + (let* ([obj-dir (format "/tmp/jerbuild-binary-~a" (get-process-id))] + [program-so (format "~a/program.so" obj-dir)] + [program-wpo (format "~a/program.wpo" obj-dir)] + [program-wp-so (format "~a/program.wp.so" obj-dir)] + [petite-hdr (format "~a/petite_boot.h" obj-dir)] + [scheme-hdr (format "~a/scheme_boot.h" obj-dir)] + [program-hdr (format "~a/program_boot.h" obj-dir)] + [main-c (format "~a/main.c" obj-dir)]) + + (printf "=== jerbuild binary build ===\n") + (printf " Entry: ~a\n" entry) + (printf " Output: ~a\n" output) + (printf " Host: ~a\n" mt) + (printf " CC: ~a\n" cc) + (printf " Chez: ~a\n\n" csv-dir) + + (system (format "mkdir -p ~a" (shell-quote obj-dir))) + + ;; Set library-directories: user libs redirect to obj-dir (so + ;; compile-imported-libraries doesn't pollute user source trees); + ;; bundle lib is plain (precompiled .so/.wpo live there already). + ;; + ;; jerbuild's own WPO image internalizes libraries reachable from + ;; (jerboa build) — (jerboa core), (std misc string), (std pregexp), etc. + ;; When user code imports any of these, Chez treats them as "already + ;; loaded" and skips compilation, so compile-whole-program needs their + ;; .wpo files on disk. build-jerbuild.sh stages them next to the .sls + ;; sources in the bundle. + (library-directories + (append (map (lambda (l) (cons l obj-dir)) user-libs) + (list bundle-lib))) + + (compile-imported-libraries #t) + (generate-wpo-files #t) + + (printf "==> [1/4] WPO compile ~a\n" entry) + (compile-program entry program-so) + (compile-whole-program program-wpo program-wp-so #t) + + (printf "==> [2/4] Embed boots + program as C arrays\n") + (embed-bytes-as-c-array petite-boot petite-hdr "petite_boot") + (embed-bytes-as-c-array scheme-boot scheme-hdr "scheme_boot") + (embed-bytes-as-c-array program-wp-so program-hdr "program_boot") + + (printf "==> [3/4] Generate main.c\n") + (call-with-output-file main-c + (lambda (port) (display *binary-main-c-template* port)) + 'replace) + + (printf "==> [4/4] Compile + link -> ~a\n" output) + (let* ([extra-archives + (apply string-append + (map (lambda (a) + (let ([p (format "~a/~a" csv-dir a)]) + (if (file-exists? p) + (string-append " " (shell-quote p)) + ""))) + '("liblz4.a" "libz.a")))] + [os-libs (machine-type->os-libs mt)] + [cc-cmd (format "~a -I~a -I~a -O2 -o ~a ~a ~a~a ~a" + cc + (shell-quote obj-dir) + (shell-quote csv-dir) + (shell-quote output) + (shell-quote main-c) + (shell-quote libkernel) + extra-archives + os-libs)]) + (printf " ~a\n" cc-cmd) + (let ([rc (system cc-cmd)]) + (unless (zero? rc) + (error 'jerbuild (format "binary: cc exited ~a" rc))))) + + (system (format "rm -rf ~a" (shell-quote obj-dir))) + (printf "\n=== Build complete: ~a ===\n" output))))))) + +;;;; ============================================================ ;;;; Entry point ;;;; ============================================================ @@ -1408,6 +1639,8 @@ (run-exec (cdr args))] [(and (pair? args) (string=? (car args) "compile")) (run-compile (cdr args))] + [(and (pair? args) (string=? (car args) "binary")) + (run-binary (cdr args))] [(and (pair? args) (string=? (car args) "transpile")) (let-values ([(src-dir lib-dir) (parse-args (cdr args))]) (jerbuild src-dir lib-dir))] --- a/support/build-jerbuild.sh +++ b/support/build-jerbuild.sh @@ -124,7 +124,8 @@ echo "" WPO_SO="${OUTPUT}.wp.so" OBJ_DIR=$(mktemp -d "/tmp/jerbuild-obj.XXXXXX") BUNDLE_TAR=$(mktemp "/tmp/jerbuild-bundle.XXXXXX.tar") -trap 'rm -rf "$OBJ_DIR" "$WPO_SO" "$BUNDLE_TAR" petite_boot.h scheme_boot.h program_boot.h bundle_tar.h bundle_meta.h "${OUTPUT}-main.c"' EXIT +BUNDLE_STAGE=$(mktemp -d "/tmp/jerbuild-stage.XXXXXX") +trap 'rm -rf "$OBJ_DIR" "$WPO_SO" "$BUNDLE_TAR" "$BUNDLE_STAGE" petite_boot.h scheme_boot.h program_boot.h bundle_tar.h bundle_meta.h "${OUTPUT}-main.c"' EXIT echo "==> [1/5] WPO compile jerbuild.ss" JERBOA_XPATCH="$JERBOA_XPATCH" "$SCHEME" --libdirs "$JERBOA_HOME/lib" \ @@ -132,13 +133,42 @@ JERBOA_XPATCH="$JERBOA_XPATCH" "$SCHEME" --libdirs "$JERBOA_HOME/lib" \ jerbuild.ss "$WPO_SO" "$OBJ_DIR" echo "" -echo "==> [2/5] Tarball lib/jerboa + lib/std (source files only)" +echo "==> [2/5] Tarball lib/jerboa + lib/std (source files only) + csv kernel files" # Only .ss and .sls — strip cached .so/.wpo build artifacts. # Use find + cpio piping so we don't depend on GNU tar features. (cd "$JERBOA_HOME" && find lib/jerboa lib/std \ \( -name '*.ss' -o -name '*.sls' \) -type f \ | sort \ | tar -cf "$BUNDLE_TAR" -T -) + +# Bundle the .so/.wpo files compiled into OBJ_DIR during step 1. These +# correspond to libraries that jerbuild's WPO image internalized (e.g. +# (jerboa core), (std misc string)). When user code in `jerbuild binary` +# imports any of these, Chez sees them as "already loaded" and skips +# compilation — but compile-whole-program still needs the .wpo on disk. +# build-boot.ss writes .so/.wpo into OBJ_DIR mirroring lib/'s relative +# layout, so OBJ_DIR has e.g. std/misc/string.{so,wpo}. Stage these +# under BUNDLE_STAGE/lib/ to match the source layout. +mkdir -p "$BUNDLE_STAGE/lib" +(cd "$OBJ_DIR" && find . \( -name '*.so' -o -name '*.wpo' \) -type f \ + ! -name 'program.so' ! -name 'program.wpo' \ + | tar -cf - -T -) \ + | (cd "$BUNDLE_STAGE/lib" && tar -xf -) +OBJDIR_COUNT=$(find "$BUNDLE_STAGE/lib" \( -name '*.so' -o -name '*.wpo' \) | wc -l | tr -d ' ') +echo " bundling $OBJDIR_COUNT .so/.wpo files from OBJ_DIR" +if [ "$OBJDIR_COUNT" -gt 0 ]; then + (cd "$BUNDLE_STAGE" && tar -rf "$BUNDLE_TAR" lib) +fi + +# Append the Chez kernel files needed by `jerbuild binary`: +# libkernel.a + scheme.h are link inputs; petite.boot/scheme.boot get re-embedded +# into generated binaries; liblz4.a/libz.a are optional companions. +mkdir -p "$BUNDLE_STAGE/csv/$MACHINE_TYPE" +for f in libkernel.a scheme.h petite.boot scheme.boot liblz4.a libz.a; do + [ -f "$CSV_DIR/$f" ] && cp "$CSV_DIR/$f" "$BUNDLE_STAGE/csv/$MACHINE_TYPE/" +done +(cd "$BUNDLE_STAGE" && tar -rf "$BUNDLE_TAR" csv) + BUNDLE_BYTES=$(wc -c < "$BUNDLE_TAR" | tr -d ' ') BUNDLE_SHA=$(shasum -a 256 "$BUNDLE_TAR" 2>/dev/null | awk '{print $1}' \ || sha256sum "$BUNDLE_TAR" | awk '{print $1}') @@ -319,12 +349,21 @@ int main(int argc, const char *argv[]) { " jerbuild compile --libdirs <p> [--wpo] FILE...\n" " # compile-imported-libs\n" " jerbuild exec --libdirs <p> FILE [args...] # load+run a Scheme script\n" + " jerbuild binary [--libdirs <p>] [--cc CC] <entry.ss> <output>\n" + " # build standalone binary\n" " jerbuild --jerboa-home # extract+print stdlib path\n" " jerbuild --version\n", stdout); return 0; } + /* The `binary` subcommand needs petite.boot/scheme.boot/libkernel.a/scheme.h + * on disk so the Scheme handler can embed + link them. Pre-extract the + * bundle and expose the path via env so the Scheme side can find them. */ + if (argc >= 2 && strcmp(argv[1], "binary") == 0) { + setenv("JERBUILD_BUNDLE_DIR", ensure_extracted(), 1); + } + Sscheme_init(NULL); Sregister_boot_file_bytes("petite", (void *)petite_boot_data, petite_boot_size);