Fix cross static binary SQLite linking
ober
aefaf65c3c188b8706b5e52ecdf4c3ce3a31f1ce
--- a/build-jcode-cross.ss +++ b/build-jcode-cross.ss @@ -27,10 +27,13 @@ (define target-arch (or (getenv "TARGET_ARCH") "amd64")) -;; Per-arch dispatch: (arch chez-machine c-toolchain rust-target output-name) +;; Per-arch dispatch: +;; (arch chez-machine c-toolchain rust-target output-name ar-tool target-env-lower target-env-upper) (define arch-table - '(("amd64" "ta6le" "x86_64-linux-musl-gcc" "x86_64-unknown-linux-musl" "jcode-linux-amd64") - ("arm64" "tarm64le" "aarch64-linux-musl-gcc" "aarch64-unknown-linux-musl" "jcode-linux-arm64"))) + '(("amd64" "ta6le" "x86_64-linux-musl-gcc" "x86_64-unknown-linux-musl" "jcode-linux-amd64" + "x86_64-linux-musl-ar" "x86_64_unknown_linux_musl" "X86_64_UNKNOWN_LINUX_MUSL") + ("arm64" "tarm64le" "aarch64-linux-musl-gcc" "aarch64-unknown-linux-musl" "jcode-linux-arm64" + "aarch64-linux-musl-ar" "aarch64_unknown_linux_musl" "AARCH64_UNKNOWN_LINUX_MUSL"))) (define arch-row (or (assoc target-arch arch-table) @@ -39,6 +42,9 @@ (define chez-machine (list-ref arch-row 1)) (define rust-target (list-ref arch-row 3)) +(define cross-ar (list-ref arch-row 5)) +(define rust-target-env-lower (list-ref arch-row 6)) +(define rust-target-env-upper (list-ref arch-row 7)) (define cross-prefix (format "~a/.chez-cross-~a" jerboa-home chez-machine)) (define xpatch (format "~a/build/chez/xc-~a/s/xpatch" jerboa-home chez-machine)) @@ -49,15 +55,23 @@ ;; jcode uses only this subset of jerboa-native-rs features: ;; tls — provider/provider.ss (rustls) -;; sqlite — (std db sqlite) via jerboa_sqlite_* ;; crypto — secrets.ss, hashing, hmac, aead, scrypt, argon2id -(define cargo-features "tls,sqlite,crypto") +;; +;; SQLite is linked through vendor/jerboa-sqlite/jerboa_sqlite_shim.c plus +;; support/sqlite-bundled. The Rust crate's sqlite feature exports a different +;; jerboa_sqlite_* surface, so enabling it here creates duplicate symbols and +;; still does not provide the int64/bytes/constants API the Scheme module uses. +(define cargo-features "tls,crypto") (define jerboa-native-a (or (getenv "JERBOA_NATIVE_A") (format "~a/jerboa-native-rs/target/~a/release/libjerboa_native.a" jerboa-home rust-target))) +(define sqlite-bundled-a + (format "support/sqlite-bundled/target/~a/release/libjcode_sqlite_bundled.a" + rust-target)) + (define cross-csv-dir (let ([lib (format "~a/lib" cross-prefix)]) (unless (file-directory? lib) @@ -88,7 +102,8 @@ (printf " xpatch: ~a~n" xpatch) (printf " cross-cc: ~a~n" cross-cc) (printf " output: ~a~n" output) -(printf " features: ~a~n~n" cargo-features) +(printf " features: ~a~n" cargo-features) +(printf " sqlite archive: ~a~n~n" sqlite-bundled-a) ;; ── Step 0: Build/rebuild jerboa-native-rs for x86_64-linux-musl ─────────── ;; libjerboa_native.a is shared across jerboa-shell, jerboa-code, etc. If a @@ -154,6 +169,30 @@ (printf " $ ~a~n" cmd) (zero? (system cmd))) +(define (build-sqlite-bundled!) + (require-file "support/sqlite-bundled/Cargo.toml") + (let* ([sqlite-dir "support/sqlite-bundled"] + [cargo-args (format "build --release --target ~a" rust-target)] + [cc-env (format "CARGO_TARGET_~a_LINKER='~a' CC_~a='~a' AR_~a='~a' " + rust-target-env-upper cross-cc + rust-target-env-lower cross-cc + rust-target-env-lower cross-ar)] + [via-rustup + (and rustup-bin-dir + (format "cd '~a' && env ~aPATH='~a':$PATH RUSTC='~a/rustc' '~a/cargo' ~a" + sqlite-dir cc-env rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] + [via-path (format "cd '~a' && env ~acargo ~a" sqlite-dir cc-env cargo-args)] + [ok? (or (and via-rustup (try-cargo-build via-rustup)) + (try-cargo-build via-path))]) + (unless ok? + (error 'build-jcode-cross + (format "failed to build bundled SQLite for ~a with ~a" + rust-target cross-cc))) + (unless (file-exists? sqlite-bundled-a) + (error 'build-jcode-cross + "cargo succeeded but bundled SQLite archive is missing" + sqlite-bundled-a)))) + (define (rebuild-native-lib!) (let* ([nrs-dir (format "~a/jerboa-native-rs" jerboa-home)] [cargo-args (format "build --release --no-default-features --features ~a --target ~a" @@ -201,6 +240,9 @@ [else (printf "==> jerboa-native-rs up to date (features=~a)~n" cargo-features)]) +(printf "==> building bundled SQLite archive (~a)~n" rust-target) +(build-sqlite-bundled!) + (printf "~n") ;; ── Step 0.5: isolate cross-compiled objects ─────────────────────────────── @@ -335,17 +377,6 @@ ;; jerboa-native (aproc) — (std os aproc) for bash tool "jerboa_aproc_spawn" "jerboa_aproc_spawn_pty" "jerboa_aproc_set_nonblock" "jerboa_aproc_killpg" "jerboa_aproc_wait4" - ;; jerboa-native (sqlite via rusqlite) — used by (std db sqlite) - "jerboa_sqlite_open" "jerboa_sqlite_close" "jerboa_sqlite_exec" - "jerboa_sqlite_prepare" "jerboa_sqlite_finalize" "jerboa_sqlite_reset" - "jerboa_sqlite_step" "jerboa_sqlite_changes" "jerboa_sqlite_errmsg" - "jerboa_sqlite_last_insert_rowid" - "jerboa_sqlite_bind_int" "jerboa_sqlite_bind_double" - "jerboa_sqlite_bind_text" "jerboa_sqlite_bind_blob" "jerboa_sqlite_bind_null" - "jerboa_sqlite_column_count" "jerboa_sqlite_column_type" - "jerboa_sqlite_column_int" "jerboa_sqlite_column_double" - "jerboa_sqlite_column_text" "jerboa_sqlite_column_blob" - "jerboa_sqlite_column_name" ;; jerboa-native (net/io) "jerboa_writev2")) @@ -426,7 +457,7 @@ (display "/* Shim function decls — jerboa_sqlite_shim.c + jcode_tui_shim.c + landlock-shim.c */\n" out) (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) ffi-shim-symbols) ;; extern decls for libjerboa_native.a symbols - (display "\n/* libjerboa_native.a — features=tls,sqlite,crypto */\n" out) + (display "\n/* libjerboa_native.a — features=tls,crypto */\n" out) (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) jerboa-native-symbols) ;; POSIX libc symbols already declared by system headers above. (display "\nextern int *__errno_location(void);\n" out) @@ -493,33 +524,17 @@ (define sqlite-shim-c "vendor/jerboa-sqlite/jerboa_sqlite_shim.c") (define jcode-tui-shim-c "src/jcode/ui/jcode_tui_shim.c") (define landlock-shim-c "support/landlock-shim.c") +(define sqlite-include-dir "support") (require-file sqlite-shim-c) (require-file jcode-tui-shim-c) (require-file landlock-shim-c) (require-file jerboa-native-a) +(require-file sqlite-bundled-a) +(require-file "support/sqlite3.h") -;; The jerboa_sqlite_shim.c calls into sqlite3_* directly (declared via -;; <sqlite3.h>). At link time, those symbols come from libjerboa_native.a -;; which has rusqlite bundling its own sqlite3. We need ANY sqlite3.h for -;; compile, since the public sqlite3 C API is stable across versions — -;; macOS Homebrew's header is fine. Find via brew. -(define brew-sqlite-prefix - (capture-line "brew --prefix sqlite 2>/dev/null")) - -(define sqlite-include-dir - (cond - [(and brew-sqlite-prefix - (file-exists? (format "~a/include/sqlite3.h" brew-sqlite-prefix))) - (format "~a/include" brew-sqlite-prefix)] - [(file-exists? "/usr/include/sqlite3.h") "/usr/include"] - [(file-exists? "/usr/local/include/sqlite3.h") "/usr/local/include"] - [else - (error 'build-jcode-cross - "no sqlite3.h found. Install via: brew install sqlite")])) - -(printf " sqlite3.h: ~a/sqlite3.h~n" sqlite-include-dir) -(printf " tls/sqlite/crypto via libjerboa_native.a~n") +(printf " sqlite3.h: support/sqlite3.h~n") +(printf " tls/crypto via libjerboa_native.a; sqlite via jerboa_sqlite_shim.c + bundled SQLite~n") ;; -Wl,--export-dynamic exports symbols into the dynamic symbol table so ;; Chez's foreign-procedure / dlsym(RTLD_DEFAULT) can find them in a @@ -535,13 +550,14 @@ "~a ~a ~a ~a " ;; main.c + 3 shims "~a/libkernel.a ~a/libz.a ~a/liblz4.a " ;; chez kernel "~a " ;; libjerboa_native.a + "~a " ;; bundled sqlite3 "-Wl,--allow-multiple-definition " "-Wl,--defsym=_dl_find_object=0 " "-lm -ldl -lpthread") cross-cc cross-csv-dir sqlite-include-dir output main-c-path sqlite-shim-c jcode-tui-shim-c landlock-shim-c cross-csv-dir cross-csv-dir cross-csv-dir - jerboa-native-a)) + jerboa-native-a sqlite-bundled-a)) (printf " ~a~n" link-cmd) (let ([rc (system link-cmd)]) (unless (zero? rc) --- a/build-jcode-freebsd-cross.ss +++ b/build-jcode-freebsd-cross.ss @@ -34,15 +34,22 @@ ;; jcode uses only this subset of jerboa-native-rs features: ;; tls — provider/provider.ss (rustls) -;; sqlite — (std db sqlite) via jerboa_sqlite_* ;; crypto — secrets.ss, hashing, hmac, aead, scrypt, argon2id -(define cargo-features "tls,sqlite,crypto") +;; +;; SQLite is linked through vendor/jerboa-sqlite/jerboa_sqlite_shim.c plus +;; support/sqlite-bundled. The Rust crate's sqlite feature exports a different +;; jerboa_sqlite_* surface, so enabling it here creates duplicate symbols and +;; still does not provide the int64/bytes/constants API the Scheme module uses. +(define cargo-features "tls,crypto") (define jerboa-native-a (or (getenv "JERBOA_NATIVE_A") (format "~a/jerboa-native-rs/target/x86_64-unknown-freebsd/release/libjerboa_native.a" jerboa-home))) +(define sqlite-bundled-a + "support/sqlite-bundled/target/x86_64-unknown-freebsd/release/libjcode_sqlite_bundled.a") + (define cross-csv-dir (let ([lib (format "~a/lib" cross-prefix)]) (unless (file-directory? lib) @@ -73,7 +80,8 @@ (printf " xpatch: ~a~n" xpatch) (printf " cross-cc: ~a~n" cross-cc) (printf " output: ~a~n" output) -(printf " features: ~a~n~n" cargo-features) +(printf " features: ~a~n" cargo-features) +(printf " sqlite archive: ~a~n~n" sqlite-bundled-a) ;; ── Step 0: Build/rebuild jerboa-native-rs for x86_64-unknown-freebsd ────── ;; libjerboa_native.a is shared across jerboa-shell, jerboa-code, etc. If a @@ -139,20 +147,40 @@ (printf " $ ~a~n" cmd) (zero? (system cmd))) +(define freebsd-cc-env + (format + "CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_LINKER='~a' CC_x86_64_unknown_freebsd='~a' AR_x86_64_unknown_freebsd=/opt/homebrew/opt/llvm/bin/llvm-ar " + cross-cc cross-cc)) + +(define (build-sqlite-bundled!) + (require-file "support/sqlite-bundled/Cargo.toml") + (let* ([sqlite-dir "support/sqlite-bundled"] + [cargo-args "build --release --target x86_64-unknown-freebsd"] + [via-rustup + (and rustup-bin-dir + (format "cd '~a' && env ~aPATH='~a':$PATH RUSTC='~a/rustc' '~a/cargo' ~a" + sqlite-dir freebsd-cc-env rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] + [via-path (format "cd '~a' && env ~acargo ~a" sqlite-dir freebsd-cc-env cargo-args)] + [ok? (or (and via-rustup (try-cargo-build via-rustup)) + (try-cargo-build via-path))]) + (unless ok? + (error 'build-jcode-freebsd-cross + (string-append "failed to build bundled SQLite for x86_64-unknown-freebsd with " + cross-cc))) + (unless (file-exists? sqlite-bundled-a) + (error 'build-jcode-freebsd-cross + "cargo succeeded but bundled SQLite archive is missing" + sqlite-bundled-a)))) + (define (rebuild-native-lib!) (let* ([nrs-dir (format "~a/jerboa-native-rs" jerboa-home)] [cargo-args (format "build --release --no-default-features --features ~a --target x86_64-unknown-freebsd" cargo-features)] - ;; cc-rs needs LLVM's llvm-ar (not Apple's /usr/bin/ar) and the - ;; FreeBSD clang wrapper. Forward these to the cargo invocation. - [cc-env (format - "CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_LINKER='~a' CC_x86_64_unknown_freebsd='~a' AR_x86_64_unknown_freebsd=/opt/homebrew/opt/llvm/bin/llvm-ar " - cross-cc cross-cc)] [via-rustup (and rustup-bin-dir (format "cd '~a' && env ~aPATH='~a':$PATH RUSTC='~a/rustc' '~a/cargo' ~a" - nrs-dir cc-env rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] - [via-path (format "cd '~a' && env ~acargo ~a" nrs-dir cc-env cargo-args)] + nrs-dir freebsd-cc-env rustup-bin-dir rustup-bin-dir rustup-bin-dir cargo-args))] + [via-path (format "cd '~a' && env ~acargo ~a" nrs-dir freebsd-cc-env cargo-args)] [ok? (or (and via-rustup (try-cargo-build via-rustup)) (try-cargo-build via-path))]) (unless ok? @@ -181,6 +209,9 @@ [else (printf "==> jerboa-native-rs up to date (features=~a)~n" cargo-features)]) +(printf "==> building bundled SQLite archive (x86_64-unknown-freebsd)~n") +(build-sqlite-bundled!) + (printf "~n") ;; ── Step 0.5: isolate cross-compiled objects ─────────────────────────────── @@ -302,13 +333,16 @@ (define jerboa-native-symbols '(;; jerboa-native (TLS/rustls) - "jerboa_tls_server_new" "jerboa_tls_server_new_mtls" + "jerboa_tls_server_new" "jerboa_tls_server_new_pem" + "jerboa_tls_server_new_mtls" "jerboa_tls_server_free" "jerboa_tls_accept" "jerboa_tls_connect" "jerboa_tls_connect_pinned" "jerboa_tls_connect_mtls" "jerboa_tls_close" "jerboa_tls_read" "jerboa_tls_write" "jerboa_tls_flush" "jerboa_tls_get_fd" "jerboa_tls_set_nonblock" "jerboa_last_error" + ;; jerboa-native (x509) — self-signed cert for the debug-REPL TLS listener + "jerboa_x509_generate_self_signed_mem" ;; jerboa-native (crypto) "jerboa_sha1" "jerboa_sha256" "jerboa_sha384" "jerboa_sha512" "jerboa_random_bytes" "jerboa_timing_safe_equal" @@ -324,17 +358,6 @@ ;; jerboa-native (aproc) — (std os aproc) for bash tool "jerboa_aproc_spawn" "jerboa_aproc_spawn_pty" "jerboa_aproc_set_nonblock" "jerboa_aproc_killpg" "jerboa_aproc_wait4" - ;; jerboa-native (sqlite via rusqlite) — used by (std db sqlite) - "jerboa_sqlite_open" "jerboa_sqlite_close" "jerboa_sqlite_exec" - "jerboa_sqlite_prepare" "jerboa_sqlite_finalize" "jerboa_sqlite_reset" - "jerboa_sqlite_step" "jerboa_sqlite_changes" "jerboa_sqlite_errmsg" - "jerboa_sqlite_last_insert_rowid" - "jerboa_sqlite_bind_int" "jerboa_sqlite_bind_double" - "jerboa_sqlite_bind_text" "jerboa_sqlite_bind_blob" "jerboa_sqlite_bind_null" - "jerboa_sqlite_column_count" "jerboa_sqlite_column_type" - "jerboa_sqlite_column_int" "jerboa_sqlite_column_double" - "jerboa_sqlite_column_text" "jerboa_sqlite_column_blob" - "jerboa_sqlite_column_name" ;; jerboa-native (net/io) — jerboa_writev2 is Linux-only (http_parse ;; module is #[cfg(target_os = "linux")] in jerboa-native-rs/src/lib.rs); ;; provided as a return-error stub via freebsd-stub-symbols below. @@ -405,7 +428,7 @@ (display "\n/* Shim function decls — jerboa_sqlite_shim.c + jcode_tui_shim.c */\n" out) (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) ffi-shim-symbols) ;; extern decls for libjerboa_native.a symbols - (display "\n/* libjerboa_native.a — features=tls,sqlite,crypto */\n" out) + (display "\n/* libjerboa_native.a — features=tls,crypto */\n" out) (for-each (lambda (n) (fprintf out "extern void ~a();\n" n)) jerboa-native-symbols) ;; register all symbols at startup (newline out) @@ -473,34 +496,18 @@ (define sqlite-shim-c "vendor/jerboa-sqlite/jerboa_sqlite_shim.c") (define jcode-tui-shim-c "src/jcode/ui/jcode_tui_shim.c") +(define sqlite-include-dir "support") ;; landlock-shim is Linux-only (sys/prctl.h, __NR_landlock_create_ruleset); ;; on FreeBSD jerboa_landlock_sandbox is provided by a static stub above. (require-file sqlite-shim-c) (require-file jcode-tui-shim-c) (require-file jerboa-native-a) +(require-file sqlite-bundled-a) +(require-file "support/sqlite3.h") -;; The jerboa_sqlite_shim.c calls into sqlite3_* directly (declared via -;; <sqlite3.h>). At link time, those symbols come from libjerboa_native.a -;; which has rusqlite bundling its own sqlite3. We need ANY sqlite3.h for -;; compile, since the public sqlite3 C API is stable across versions — -;; macOS Homebrew's header is fine. Find via brew. -(define brew-sqlite-prefix - (capture-line "brew --prefix sqlite 2>/dev/null")) - -(define sqlite-include-dir - (cond - [(and brew-sqlite-prefix - (file-exists? (format "~a/include/sqlite3.h" brew-sqlite-prefix))) - (format "~a/include" brew-sqlite-prefix)] - [(file-exists? "/usr/include/sqlite3.h") "/usr/include"] - [(file-exists? "/usr/local/include/sqlite3.h") "/usr/local/include"] - [else - (error 'build-jcode-freebsd-cross - "no sqlite3.h found. Install via: brew install sqlite")])) - -(printf " sqlite3.h: ~a/sqlite3.h~n" sqlite-include-dir) -(printf " tls/sqlite/crypto via libjerboa_native.a~n") +(printf " sqlite3.h: support/sqlite3.h~n") +(printf " tls/crypto via libjerboa_native.a; sqlite via jerboa_sqlite_shim.c + bundled SQLite~n") ;; FreeBSD link differences from Linux musl: ;; - Dynamic, not static (FreeBSD libc symbol versioning blocks libc.a) @@ -520,11 +527,12 @@ "~a ~a ~a " ;; main.c + 2 shims "~a/libkernel.a ~a/libz.a ~a/liblz4.a " ;; chez kernel "~a " ;; libjerboa_native.a + "~a " ;; bundled sqlite3 "-lm -lpthread -lutil") cross-cc cross-csv-dir sqlite-include-dir output main-c-path sqlite-shim-c jcode-tui-shim-c cross-csv-dir cross-csv-dir cross-csv-dir - jerboa-native-a)) + jerboa-native-a sqlite-bundled-a)) (printf " ~a~n" link-cmd) (let ([rc (system link-cmd)]) (unless (zero? rc) --- a/support/jcode-main.c +++ b/support/jcode-main.c @@ -12,9 +12,16 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <unistd.h> #include <fcntl.h> #include <sys/types.h> +/* ffi_symbols.h registers libc's syscall symbol for static Linux sandboxing. + * macOS <unistd.h> also declares syscall(int, ...); jerbuild currently emits + * extern void syscall(), which conflicts with that prototype. Keep the useful + * POSIX declarations but hide only syscall's declaration from this translation + * unit before including the generated FFI header below. */ +#define syscall jcode_libc_syscall +#include <unistd.h> +#undef syscall #include "petite_boot.h" #include "scheme_boot.h" @@ -25,11 +32,13 @@ static const char *write_program_tmpfile(void) { static char path[] = "/tmp/jcode-prog-XXXXXX"; int fd = mkstemp(path); if (fd < 0) { perror("mkstemp"); exit(1); } - ssize_t n = write(fd, program_boot_data, program_boot_size); - if (n != (ssize_t)program_boot_size) { - perror("write"); close(fd); unlink(path); exit(1); + FILE *out = fdopen(fd, "wb"); + if (!out) { perror("fdopen"); close(fd); unlink(path); exit(1); } + size_t n = fwrite(program_boot_data, 1, program_boot_size, out); + if (n != program_boot_size) { + perror("fwrite"); fclose(out); unlink(path); exit(1); } - close(fd); + if (fclose(out) != 0) { perror("fclose"); unlink(path); exit(1); } return path; }