Fix cross-build harness so jgl-linux-amd64 actually runs
ober
c9b3b41e7e9fd659a7ef8c478fd1b8be028ee3ce
--- a/build-jgl-cross.ss +++ b/build-jgl-cross.ss @@ -2,13 +2,20 @@ ;;; build-jgl-cross.ss — Cross-compile jgl from macOS arm64 to Linux x86_64 musl. ;;; ;;; jgl's HTTPS client uses (std net request) -> (std net tls-rustls), i.e. the -;;; Rust `jerboa-native` crate (rustls). So the cross binary must link a -;;; musl-cross build of that crate and register its jerboa_* FFI symbols — the -;;; same backend as the native `jerbuild build`. (The jerboa-ssl/OpenSSL shim -;;; path used by older jerboa-* cross scripts only provides jerboa_ssl_*/ -;;; jerboa_tcp_* and would fail at runtime on jerboa_tls_connect.) +;;; Rust `jerboa-native` crate (rustls). So the cross binary links a musl-cross +;;; build of that crate and registers its jerboa_* FFI symbols — the same TLS +;;; backend as the native `jerbuild build`. ;;; -;;; Driven by `make cross-linux`, which first cargo-builds the crate for +;;; The C entry point mirrors jerboa-code's working harness: +;;; - set JERBOA_STATIC=1 so (std net)/process use registered libc symbols; +;;; - Sbuild_heap FIRST, THEN Sforeign_symbol registration (registering before +;;; the heap exists segfaults — that was the jerboa-aws cross-script bug); +;;; - register both jerboa_* (from the crate) and the libc symbols (std net +;;; tcp/process call socket/getaddrinfo/... by name, unresolvable via dlsym +;;; in a -static binary); +;;; - forward argv straight to Sscheme_program. +;;; +;;; Driven by `make cross-linux`, which cargo-builds the crate for ;;; x86_64-unknown-linux-musl and writes the symbol list. Inputs via env: ;;; JGL_NATIVE_A path to musl libjerboa_native.a ;;; JGL_FFI_SYMBOLS file of jerboa_* symbols (one per line) to register @@ -18,8 +25,7 @@ ;;; Prerequisites (shared across the jerboa-* family): ;;; - $JERBOA_HOME/.chez-cross-ta6le/ cross-built Chez (ta6le) ;;; - $JERBOA_HOME/build/chez/xc-ta6le/s/xpatch host -> ta6le emit loader -;;; - x86_64-linux-musl-gcc C compile + static link -;;; - rustup target x86_64-unknown-linux-musl (the Makefile builds the .a) +;;; - x86_64-linux-musl-gcc; rustup target x86_64-unknown-linux-musl ;;; ;;; Produces: jgl-linux-amd64 (static Linux x86_64 ELF). @@ -66,7 +72,7 @@ (require-file native-a) (require-file ffi-symbols-file) -;; Read the jerboa_* symbols to register with Sforeign_symbol. +;; jerboa_* symbols to register with Sforeign_symbol (one bare name per line). (define jerboa-symbols (call-with-input-file ffi-symbols-file (lambda (p) @@ -82,17 +88,28 @@ (if (and (>= (string-length s) 7) (string=? (substring s 0 7) "jerboa_")) (loop (cons s acc)) (loop acc))))))))) - -(printf "==> build-jgl-cross (rustls/musl)~n") -(printf " JERBOA_HOME: ~a~n" jerboa-home) -(printf " cross csv-dir: ~a~n" cross-csv-dir) -(printf " cross-cc: ~a~n" cross-cc) -(printf " native .a: ~a~n" native-a) -(printf " ffi symbols: ~a (~a symbols)~n" ffi-symbols-file (length jerboa-symbols)) -(printf " output: ~a~n~n" output) (when (null? jerboa-symbols) (error 'build-jgl-cross "no jerboa_* symbols found in" ffi-symbols-file)) +;; libc symbols (std net tcp + std misc process call these by name; a -static +;; binary can't dlsym them, so register them explicitly). From jerboa-code. +(define libc-symbols + '("socket" "bind" "listen" "accept" "connect" "close" "setsockopt" "getsockopt" + "getsockname" "htons" "ntohs" "inet_pton" "inet_addr" "read" "write" + "recvfrom" "sendto" "getaddrinfo" "freeaddrinfo" "inet_ntop" + "fork" "waitpid" "kill" "getpid" "getppid" "getpgid" "setpgid" "setsid" + "getuid" "geteuid" "getgid" "getegid" + "sigemptyset" "sigfillset" "sigaddset" "sigdelset" "sigismember" "sigprocmask" + "isatty" "tcgetattr" "tcsetattr" "tcgetpgrp" "tcsetpgrp" + "pipe" "dup" "dup2" "lseek" "setenv" "unsetenv" "strerror" "localtime" + "strftime" "sysconf" "getpagesize" "getrlimit" "system" "getenv" "putenv" + "_exit" "exit" "execvp" "execve" "mkstemp" "mkdtemp" "unlink" "rmdir" + "__errno_location" "fcntl")) + +(printf "==> build-jgl-cross (rustls/musl)~n") +(printf " cross csv-dir: ~a~n native .a: ~a~n symbols: ~a jerboa_* + ~a libc~n output: ~a~n~n" + cross-csv-dir native-a (length jerboa-symbols) (length libc-symbols) output) + ;; ── Stage 1: load xpatch (target=ta6le emit mode) ────────────────────────── (define orig-libdirs (library-directories)) (printf "==> [1/6] loading xpatch (compiler -> ta6le emit mode)~n") @@ -102,30 +119,28 @@ (list (cons (format "~a/lib" jgl-repo) (format "~a/lib" jgl-repo)) (cons (format "~a/lib" jerboa-home) (format "~a/lib" jerboa-home))) orig-libdirs)) - (compile-imported-libraries #t) (generate-wpo-files #t) -;; ── Stage 2: compile-program jgl.ss ───────────────────────────────────────── +;; ── Stage 2: compile-program ──────────────────────────────────────────────── (printf "==> [2/6] compile-program ~a~n" entry-script) (compile-program entry-script) (define entry-wpo - (let ([n (string-length entry-script)]) - (string-append (substring entry-script 0 (- n 3)) ".wpo"))) + (string-append (substring entry-script 0 (- (string-length entry-script) 3)) ".wpo")) -;; ── Stage 3: compile-whole-program → wpo .so ─────────────────────────────── +;; ── Stage 3: whole-program optimize → single .so ──────────────────────────── (define wpo-output (string-append output ".wp.so")) -(printf "==> [3/6] compile-whole-program ~a -> ~a~n" entry-wpo wpo-output) +(printf "==> [3/6] compile-whole-program -> ~a~n" wpo-output) (compile-whole-program entry-wpo wpo-output #t) -;; ── Stage 4: embed boot files + program as C arrays ──────────────────────── -(define (embed-as-c-array in-path var-name out-path) +;; ── Stage 4: embed boots + program as C arrays ────────────────────────────── +(define (embed in-path var out-path) (let* ([bv (call-with-port (open-file-input-port in-path) get-bytevector-all)] [n (bytevector-length bv)]) (call-with-port (open-file-output-port out-path (file-options no-fail) (buffer-mode block) (native-transcoder)) (lambda (out) - (display (format "static const unsigned char ~a[] = {\n" var-name) out) + (display (format "static const unsigned char ~a[] = {\n" var) out) (let loop ([i 0]) (when (< i n) (display (format "0x~2,'0x," (bytevector-u8-ref bv i)) out) @@ -133,76 +148,70 @@ (loop (+ i 1)))) (when (positive? n) (newline out)) (display "};\n" out) - (display (format "static const unsigned int ~a_size = sizeof(~a);\n" var-name var-name) out))) - (printf " embed ~a (~a bytes) -> ~a~n" in-path n out-path))) - -(printf "==> [4/6] embed boot files + program as C arrays~n") -(embed-as-c-array (format "~a/petite.boot" cross-csv-dir) "petite_boot" "petite_boot.h") -(embed-as-c-array (format "~a/scheme.boot" cross-csv-dir) "scheme_boot" "scheme_boot.h") -(embed-as-c-array wpo-output "jgl_program" "jgl_program.h") - -;; ── Stage 5: generate main.c (registers the crate's jerboa_* symbols) ─────── + (display (format "static const unsigned int ~a_size = sizeof(~a);\n" var var) out))) + (printf " embed ~a (~a bytes)~n" in-path n))) +(printf "==> [4/6] embed boots + program~n") +(embed (format "~a/petite.boot" cross-csv-dir) "petite_boot" "petite_boot.h") +(embed (format "~a/scheme.boot" cross-csv-dir) "scheme_boot" "scheme_boot.h") +(embed wpo-output "jgl_program" "jgl_program.h") + +;; ── Stage 5: generate main.c (jerboa-code's working harness) ──────────────── (define main-c-path (string-append output "-main.c")) -(printf "==> [5/6] generate ~a (~a jerboa_* symbols)~n" main-c-path (length jerboa-symbols)) - -(define (write-main-c) - (call-with-port - (open-file-output-port main-c-path - (file-options no-fail) (buffer-mode block) (native-transcoder)) - (lambda (out) - (display "/* Auto-generated by build-jgl-cross.ss. Do not edit. */\n" out) - (display "#define _GNU_SOURCE\n" out) - (display "#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n" out) - (display "#include <unistd.h>\n#include <sys/mman.h>\n#include <fcntl.h>\n" out) - (display "#include \"scheme.h\"\n#include \"petite_boot.h\"\n" out) - (display "#include \"scheme_boot.h\"\n#include \"jgl_program.h\"\n\n" out) - - ;; dlopen stubs (musl-static has no dlopen); FFI uses the Sforeign_symbol table. - (display "void *dlopen(const char *f, int g){ (void)f;(void)g; return (void*)1; }\n" out) - (display "void *dlsym(void *h, const char *s){ (void)h;(void)s; return NULL; }\n" out) - (display "int dlclose(void *h){ (void)h; return 0; }\n" out) - (display "static char dlerr[] = \"static binary: dlopen stubbed\";\n" out) - (display "char *dlerror(void){ return dlerr; }\n\n" out) - - (display "/* jerboa-native (rustls/crypto) exports — registered below. */\n" out) - (for-each (lambda (sym) (display (format "extern void ~a(void);\n" sym) out)) - jerboa-symbols) - (newline out) - - (display "int main(int argc, char *argv[]) {\n" out) - (display " char countbuf[32];\n" out) - (display " snprintf(countbuf, sizeof(countbuf), \"%d\", argc - 1);\n" out) - (display " setenv(\"JGL_ARGC\", countbuf, 1);\n" out) - (display " for (int i = 1; i < argc; i++) {\n" out) - (display " char name[32];\n" out) - (display " snprintf(name, sizeof(name), \"JGL_ARG%d\", i - 1);\n" out) - (display " setenv(name, argv[i], 1);\n" out) - (display " }\n\n" out) - - (display " int fd = memfd_create(\"jgl-program\", MFD_CLOEXEC);\n" out) - (display " if (fd < 0) { perror(\"memfd_create\"); return 1; }\n" out) - (display " if (write(fd, jgl_program, jgl_program_size) != (ssize_t)jgl_program_size) {\n" out) - (display " perror(\"write memfd\"); close(fd); return 1;\n" out) - (display " }\n" out) - (display " char prog_path[64];\n" out) - (display " snprintf(prog_path, sizeof(prog_path), \"/proc/self/fd/%d\", fd);\n\n" out) - - (display " Sscheme_init(NULL);\n" out) - (display " Sregister_boot_file_bytes(\"petite\", (void*)petite_boot, petite_boot_size);\n" out) - (display " Sregister_boot_file_bytes(\"scheme\", (void*)scheme_boot, scheme_boot_size);\n\n" out) - - (display " /* Register jerboa-native symbols so a -static binary resolves\n" out) - (display " * (foreign-procedure \"jerboa_*\" ...) without dlsym. */\n" out) - (for-each (lambda (sym) - (display (format " Sforeign_symbol(\"~a\", (void*)~a);\n" sym sym) out)) - jerboa-symbols) - (newline out) - - (display " Sbuild_heap(NULL, NULL);\n" out) - (display " const char *script_args[] = { argv[0] };\n" out) - (display " int status = Sscheme_program(prog_path, 1, script_args);\n" out) - (display " close(fd);\n Sscheme_deinit();\n return status;\n}\n" out)))) -(write-main-c) +(printf "==> [5/6] generate ~a (~a jerboa_* + ~a libc symbols)~n" + main-c-path (length jerboa-symbols) (length libc-symbols)) +(call-with-port + (open-file-output-port main-c-path (file-options no-fail) (buffer-mode block) (native-transcoder)) + (lambda (out) + (display "/* Auto-generated by build-jgl-cross.ss. Do not edit. */\n#define _GNU_SOURCE\n" out) + (for-each (lambda (h) (display (format "#include <~a>\n" h) out)) + '("stdlib.h" "string.h" "stdio.h" "unistd.h" "sys/mman.h" "sys/types.h" + "sys/stat.h" "sys/wait.h" "sys/resource.h" "sys/socket.h" "netinet/in.h" + "arpa/inet.h" "netdb.h" "termios.h" "fcntl.h" "signal.h" "time.h" "errno.h")) + (display "#include \"scheme.h\"\n#include \"petite_boot.h\"\n" out) + (display "#include \"scheme_boot.h\"\n#include \"jgl_program.h\"\n\n" out) + + ;; dlopen stubs: succeed only for the main exe (NULL/"") so the Scheme + ;; (load-shared-object "libfoo") guard falls back to registered symbols. + (display "void *dlopen(const char *f, int flags){ (void)flags; return (!f||f[0]=='\\0')?(void*)1:NULL; }\n" out) + (display "void *dlsym(void *h, const char *s){ (void)h;(void)s; return NULL; }\n" out) + (display "int dlclose(void *h){ (void)h; return 0; }\n" out) + (display "static char dlerr[]=\"static binary: dlopen of named libraries is stubbed\";\n" out) + (display "char *dlerror(void){ return dlerr; }\n\n" out) + + (display "/* libjerboa_native.a (rustls/crypto) exports. */\n" out) + (for-each (lambda (s) (display (format "extern void ~a();\n" s) out)) jerboa-symbols) + (display "extern int *__errno_location(void);\n\n" out) + + (display "static void register_ffi_symbols(void) {\n" out) + (for-each (lambda (s) (display (format " Sforeign_symbol(\"~a\", (void*)~a);\n" s s) out)) + jerboa-symbols) + (for-each (lambda (s) (display (format " Sforeign_symbol(\"~a\", (void*)~a);\n" s s) out)) + libc-symbols) + (display "}\n\n" out) + + (display "int main(int argc, char *argv[]) {\n" out) + (display " setenv(\"JERBOA_STATIC\", \"1\", 1);\n" out) + (display " int fd = memfd_create(\"jgl-program\", MFD_CLOEXEC);\n" out) + (display " char prog_path[64];\n" out) + (display " if (fd >= 0) {\n" out) + (display " if (write(fd, jgl_program, jgl_program_size) != (ssize_t)jgl_program_size) {\n" out) + (display " perror(\"write memfd\"); close(fd); return 1; }\n" out) + (display " snprintf(prog_path, sizeof(prog_path), \"/proc/self/fd/%d\", fd);\n" out) + (display " } else {\n" out) + (display " const char *tmp = getenv(\"TMPDIR\"); if (!tmp) tmp = \"/tmp\";\n" out) + (display " snprintf(prog_path, sizeof(prog_path), \"%s/.jgl-prog-%d.so\", tmp, getpid());\n" out) + (display " FILE *fp = fopen(prog_path, \"wb\"); if (!fp) { perror(\"fopen\"); return 1; }\n" out) + (display " if (fwrite(jgl_program, 1, jgl_program_size, fp) != jgl_program_size) {\n" out) + (display " perror(\"fwrite\"); fclose(fp); unlink(prog_path); return 1; }\n" out) + (display " fclose(fp);\n }\n" out) + (display " Sscheme_init(NULL);\n" out) + (display " Sregister_boot_file_bytes(\"petite\", (void*)petite_boot, petite_boot_size);\n" out) + (display " Sregister_boot_file_bytes(\"scheme\", (void*)scheme_boot, scheme_boot_size);\n" out) + (display " Sbuild_heap(NULL, NULL);\n" out) + (display " register_ffi_symbols();\n" out) + (display " int status = Sscheme_program(prog_path, argc, (const char **)argv);\n" out) + (display " if (fd >= 0) close(fd); else unlink(prog_path);\n" out) + (display " Sscheme_deinit();\n return status;\n}\n" out))) ;; ── Stage 6: compile + link with cross-cc ────────────────────────────────── (printf "==> [6/6] compile + link with ~a~n" cross-cc) @@ -220,5 +229,4 @@ (printf " ~a~n" link-cmd) (unless (zero? (system link-cmd)) (error 'build-jgl-cross "link failed")) - (printf "~n==> done: ~a~n" output)