multicall: on-target self-build (jerbuild binary) from a cross binary
ober
621ea00da0d4ef87532577f0b382760f2d3703d4
--- a/Makefile +++ b/Makefile @@ -229,11 +229,13 @@ jerboa-smoke: jerboa # four modes runnable on that target. The xpatch retargets the host Chez's # codegen; the cross Chez supplies target boot files + libkernel.a. # -# LIMITATION: on-target `jerbuild binary` does NOT work from a cross binary yet. -# Chez cross-compilation emits host-tagged .wpo (only .so get the target tag), -# and target-tagged .wpo can only be produced natively ON the target. The -# running multicall image has the stdlib internalized, so it cannot regenerate -# them in-process. Enabling self-build needs a bare-Chez subprocess mode (TODO). +# On-target `jerbuild binary` works from a cross binary. Chez cross-compilation +# emits host-tagged .wpo (only .so get the target tag), and target-tagged .wpo +# can only be produced natively ON the target — so jerbuild re-invokes itself in +# bare-Chez mode (`<self> scheme <helper>`) to recompile the bundled stdlib +# sources to native .wpo in a fresh process with an empty library table. The +# produced binary's link libs are recorded in the bundle (csv/<mt>/os-libs: +# static + -no-pie on linux) so a cross target's native cc links cleanly. # # Generic form (builds the cross Chez on demand via chez-cross): # make jerboa-cross CHEZ_TARGET_MACHINE=ta6le CROSS_CC=x86_64-linux-musl-gcc --- a/jerbuild.ss +++ b/jerbuild.ss @@ -1802,15 +1802,29 @@ int main(int argc, const char *argv[]) { (substring p i (string-length p))] [else (loop (- i 1))]))) -;; Run compile-program + compile-whole-program in a fresh Chez subprocess so -;; the cross target's recompiled libraries do not collide with the libraries -;; already loaded into jerbuild's own host image. +;; Command that runs the WPO helper script in a fresh process with an empty +;; library table. Inside the multicall binary (JERBOA_SELF_EXE set) we +;; re-invoke ourselves in bare-Chez mode (`<self> scheme <helper>`), since no +;; external `scheme` exists on a target host; otherwise use a stock external +;; scheme (`scheme --quiet --script <helper>`). +(define (wpo-subprocess-command helper) + (let ([self (getenv "JERBOA_SELF_EXE")]) + (if (and self (file-exists? self)) + (format "~a scheme ~a" (shell-quote self) (shell-quote helper)) + (format "~a --quiet --script ~a" + (shell-quote (find-scheme-binary)) (shell-quote helper))))) + +;; Run compile-program + compile-whole-program in a fresh Chez subprocess so a +;; clean library table is used. Needed (a) for cross builds, where xpatch +;; retargets codegen and recompiled target libraries must not collide with +;; jerbuild's host image; and (b) inside the multicall binary, whose running +;; image has the stdlib internalized — only a fresh process can recompile the +;; bundled sources to native .wpo. xpatch #f => native (no retarget). (define (cross-wpo-subprocess! entry obj-dir bundle-lib user-libs xpatch program-so program-wpo program-wp-so) - (let* ([scheme (find-scheme-binary)] - [helper (format "~a/cross-wpo.ss" obj-dir)] - ;; Redirect bundle-lib to obj-dir/bundle so freshly compiled target - ;; .so files don't overwrite the host .so files in the bundle. + (let* ([helper (format "~a/cross-wpo.ss" obj-dir)] + ;; Redirect bundle-lib to obj-dir/bundle so freshly compiled + ;; .so files don't overwrite the .so files in the bundle. [bundle-redirect (string-append obj-dir "/bundle")] [libdirs (append @@ -1823,9 +1837,10 @@ int main(int argc, const char *argv[]) { (lambda (port) (display "(import (chezscheme))\n" port) (fprintf port "(library-directories '~s)\n" libdirs) - ;; xpatch may mutate library-directories; restore after loading. - (fprintf port "(load ~s)\n" xpatch) - (fprintf port "(library-directories '~s)\n" libdirs) + (when xpatch + ;; xpatch may mutate library-directories; restore after loading. + (fprintf port "(load ~s)\n" xpatch) + (fprintf port "(library-directories '~s)\n" libdirs)) (display "(compile-imported-libraries #t)\n" port) (display "(generate-wpo-files #t)\n" port) (fprintf port "(compile-program ~s ~s)\n" entry program-so) @@ -1833,14 +1848,14 @@ int main(int argc, const char *argv[]) { program-wpo program-wp-so) (display "(exit 0)\n" port)) 'replace) - (printf " scheme: ~a\n" scheme) - (printf " helper: ~a\n" helper) - (let ([rc (system (format "~a --quiet --script ~a" - (shell-quote scheme) (shell-quote helper)))]) - (unless (zero? rc) - (error 'jerbuild - (format "cross WPO subprocess failed (rc=~a; helper=~a)" - rc helper)))))) + (let ([cmd (wpo-subprocess-command helper)]) + (printf " helper: ~a\n" helper) + (printf " cmd: ~a\n" cmd) + (let ([rc (system cmd)]) + (unless (zero? rc) + (error 'jerbuild + (format "WPO subprocess failed (rc=~a; helper=~a)" + rc helper))))))) (define (do-binary-build libs cc-arg extra-archives extra-sources extra-ldflags rust-crates main-c-override @@ -1970,9 +1985,17 @@ int main(int argc, const char *argv[]) { ;; "attempting to re-install compile-time part of library" because they ;; are already loaded. Run the WPO step in a fresh Chez subprocess so it ;; sees a clean library table. + ;; Run WPO in a fresh subprocess when (a) cross (xpatch retargets), or + ;; (b) we are the multicall binary (JERBOA_SELF_EXE set): our running + ;; image has the stdlib internalized, so an in-process compile-whole- + ;; program would need their .wpo on disk, and a cross bundle's .wpo are + ;; the build host's machine-type. A fresh bare-Chez process recompiles + ;; bundled sources to native .wpo. Plain `scheme --script jerbuild.ss` + ;; keeps the in-process path (its bundle .wpo already match the host). (cond - [xpatch - (printf "==> [1/5] WPO compile in isolated subprocess (xpatch=~a)\n" xpatch) + [(or xpatch (getenv "JERBOA_SELF_EXE")) + (printf "==> [1/5] WPO compile in isolated subprocess~a\n" + (if xpatch (format " (xpatch=~a)" xpatch) " (multicall self)")) (cross-wpo-subprocess! entry obj-dir bundle-lib user-libs xpatch program-so program-wpo program-wp-so)] [else @@ -2077,6 +2100,12 @@ int main(int argc, const char *argv[]) { chez-archives (join-raw extra-ldflags) (or os-libs-override + (let ([f (format "~a/os-libs" csv-dir)]) + (and (file-exists? f) + (let ([line (call-with-input-file f get-line)]) + (and (string? line) + (> (string-length line) 0) + line)))) (machine-type->os-libs mt)))]) (printf " ~a\n" cc-cmd) (let ([rc (system cc-cmd)]) --- a/support/build-jerboa-multicall.ss +++ b/support/build-jerboa-multicall.ss @@ -341,6 +341,36 @@ (compile-program entry-ss (format "~a/program.so" obj-dir)) (compile-whole-program (format "~a/program.wpo" obj-dir) program-wp-so #t) +;; OS link libs for the FINAL cc link of the multicall binary. Cross Chez is +;; configured --disable-curses --disable-x11 --disable-iconv, so libkernel.a +;; has no ncurses/iconv refs; native host Chez is built --as-is. +(define os-libs + (if cross? + (case target-os + [(linux) "-lm -ldl -lpthread -static"] + [(freebsd) "-lm -lpthread"] + [(darwin) "-lm -lpthread"] + [else "-lm -lpthread"]) + (cond [(string-suffix? machine "osx") "-lm -lpthread -lncurses -liconv"] + [(string-suffix? machine "fb") "-lm -lpthread -lncurses -L/usr/local/lib -liconv"] + [else "-lm -ldl -lpthread -lncurses"]))) +;; Libs jerbuild must pass when it links an ON-TARGET binary against this +;; bundle's libkernel.a. Recorded as csv/<machine>/os-libs so jerbuild overrides +;; its machine-type guess, which assumes a curses/iconv-enabled host Chez and +;; would wrongly add -lncurses/-liconv on a cross target's bare toolchain. +;; libkernel.a is built non-PIC, but a target's native cc may default to PIE +;; (e.g. Alpine gcc), which rejects R_X86_64_32 relocations. Link the produced +;; binary -static on linux (matches the multicall binary; proven on Alpine); +;; freebsd can't go -static (libc symbol versioning) so force -no-pie instead. +(define produced-os-libs + (if cross? + (case target-os + [(linux) "-lm -ldl -lpthread -static -no-pie"] + [(freebsd) "-lm -lpthread -no-pie"] + [(darwin) "-lm -lpthread"] + [else "-lm -lpthread"]) + os-libs)) + (printf "==> [3/6] tarball stdlib + obj + kernel for jerbuild bundle~n") (define bundle-tar (format "~a/bundle.tar" build-dir)) (define stage (format "~a/stage" build-dir)) @@ -355,6 +385,8 @@ (when (file-exists? (format "~a/~a" csv-dir f)) (run (format "cp ~a/~a ~a/csv/~a/" (shell-quote csv-dir) f (shell-quote stage) machine)))) '("libkernel.a" "scheme.h" "petite.boot" "scheme.boot" "liblz4.a" "libz.a")) +(call-with-output-file (format "~a/csv/~a/os-libs" stage machine) + (lambda (o) (display produced-os-libs o) (newline o)) 'replace) (run (format "cd ~a && tar -rf ~a csv" (shell-quote stage) (shell-quote bundle-tar))) (define sha-file (format "~a/bundle.sha256" build-dir)) @@ -371,18 +403,6 @@ (lambda (o) (fprintf o "static const char bundle_sha256[] = \"~a\";~n" bundle-sha)) 'replace) (printf "==> [5/6] compile + link -> ~a~n" output) -;; Cross Chez is configured --disable-curses --disable-x11 --disable-iconv, so -;; libkernel.a has no ncurses/iconv refs; native host Chez is built --as-is. -(define os-libs - (if cross? - (case target-os - [(linux) "-lm -ldl -lpthread -static"] - [(freebsd) "-lm -lpthread"] - [(darwin) "-lm -lpthread"] - [else "-lm -lpthread"]) - (cond [(string-suffix? machine "osx") "-lm -lpthread -lncurses -liconv"] - [(string-suffix? machine "fb") "-lm -lpthread -lncurses -L/usr/local/lib -liconv"] - [else "-lm -ldl -lpthread -lncurses"]))) (define extra-archives (apply string-append (map (lambda (a) (if (file-exists? (format "~a/~a" csv-dir a)) (format " ~a/~a" csv-dir a) "")) --- a/support/multicall-main.c +++ b/support/multicall-main.c @@ -27,6 +27,13 @@ #include <sys/types.h> #include <unistd.h> +#if defined(__APPLE__) +#include <mach-o/dyld.h> +#include <stdint.h> +#elif defined(__FreeBSD__) +#include <sys/sysctl.h> +#endif + #include "petite_boot.h" #include "scheme_boot.h" #include "program_boot.h" @@ -147,6 +154,27 @@ static const char *basename_of(const char *p) { return s ? s + 1 : p; } +/* Absolute path to this executable, so `jerbuild binary` can re-invoke itself + * in bare-Chez mode for the WPO subprocess. Exported as JERBOA_SELF_EXE. */ +static int self_exe_path(char *buf, size_t n) { +#if defined(__APPLE__) + uint32_t sz = (uint32_t)n; + char tmp[1024]; + if (_NSGetExecutablePath(tmp, &sz) != 0) return -1; + if (!realpath(tmp, buf)) { strncpy(buf, tmp, n - 1); buf[n - 1] = '\0'; } + return 0; +#elif defined(__FreeBSD__) + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; + size_t sz = n; + return sysctl(mib, 4, buf, &sz, NULL, 0); +#else /* Linux and other /proc systems */ + ssize_t r = readlink("/proc/self/exe", buf, n - 1); + if (r < 0) return -1; + buf[r] = '\0'; + return 0; +#endif +} + static const char JERBUILD_USAGE[] = "Usage:\n" " jerbuild <src> <lib> # transpile .ss -> .sls\n" @@ -174,6 +202,22 @@ int main(int argc, const char *argv[]) { if (!strcmp(mode, "mcp")) mode = "jmcp"; if (!strcmp(mode, "lsp")) mode = "jlsp"; + /* Bare-Chez mode: `jerboa scheme [script [args...]]` boots petite+scheme + * WITHOUT the embedded program image, behaving as a stock `scheme`. Used by + * `jerbuild binary` on a target host to run the WPO compile in a fresh + * process with an empty library table, so bundled stdlib sources recompile + * to native .wpo (cross-built .wpo carry the build host's machine-type). */ + if (!strcmp(name, "jerboa") && argc > 1 && !strcmp(argv[1], "scheme")) { + Sscheme_init(NULL); + Sregister_boot_file_bytes("petite", (void *)petite_boot_data, petite_boot_size); + Sregister_boot_file_bytes("scheme", (void *)scheme_boot_data, scheme_boot_size); + Sbuild_heap(NULL, register_symbols); + int st = (argc > 2) ? Sscheme_script(argv[2], argc - 2, argv + 2) + : Sscheme_start(argc - 1, argv + 1); + Sscheme_deinit(); + return st; + } + /* jerbuild mode: C-side fast paths that avoid booting Chez, plus bundle * pre-extraction for the subcommands that link binaries. a1 is the first * real jerbuild argument (the token after the mode selector). */ @@ -193,6 +237,11 @@ int main(int argc, const char *argv[]) { setenv("JERBUILD_BUNDLE_DIR", ensure_extracted(), 1); } + { + static char selfexe[1024]; + if (self_exe_path(selfexe, sizeof selfexe) == 0) + setenv("JERBOA_SELF_EXE", selfexe, 1); + } setenv("JERBOA_MULTICALL_NAME", mode, 1); Sscheme_init(NULL);