Fix static Qt vendored build
ober
f84469097e11f2d0e08c1a345cbc1090de1e9f3c
--- a/Dockerfile +++ b/Dockerfile @@ -253,7 +253,7 @@ RUN mkdir -p /opt/tree-sitter-grammars && \ build_grammar scheme 6cdh/tree-sitter-scheme main && \ echo "Grammars built:" && ls /opt/tree-sitter-grammars/ -# ── Phase 6: Gerbil/Chez dependencies ────────────────────────────────── +# ── Phase 6: Jerboa/Chez dependencies ────────────────────────────────── ENV PKG_CONFIG_PATH=/opt/qt6-static/lib/pkgconfig ENV SCHEME=/opt/chez/bin/scheme @@ -268,19 +268,29 @@ COPY vendor/qt_shim.cpp vendor/qt_shim.h /deps/qt-shim/ # qt_chez_shim.c is maintained in vendor/ (not in chez-qt) — copy it in COPY vendor/qt_chez_shim.c vendor/qt_shim.h /deps/chez-qt/ +# jerboa-shell keeps these support libraries as jerbuild input files. +# Transpile only the libraries imported by support/static-precompile-deps.ss. +RUN mkdir -p /tmp/jsh-jerbuild-src/compat /tmp/jsh-jerbuild-src/jsh && \ + cp /deps/jsh/jerboa-src/src/compat/gambit.ss /tmp/jsh-jerbuild-src/compat/ && \ + cp /deps/jsh/jerboa-src/src/jsh/ffi.ss /tmp/jsh-jerbuild-src/jsh/ && \ + cp /deps/jsh/jerboa-src/src/jsh/static-compat.ss /tmp/jsh-jerbuild-src/jsh/ && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ + --script /deps/jsh/support/jerbuild.ss /tmp/jsh-jerbuild-src /deps/jsh/src --force && \ + rm -rf /tmp/jsh-jerbuild-src + # Pre-compile all Chez library dependencies. # These .so files are baked into the image so jemacs builds only # need to compile jemacs-specific modules. COPY support/static-precompile-deps.ss /tmp/static-precompile-deps.ss RUN /opt/chez/bin/scheme \ - --libdirs /deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/chez-pcre2:/deps/chez-scintilla/src:/deps/chez-qt \ + --libdirs /deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/chez-pcre2:/deps/chez-scintilla/lib:/deps/chez-qt/lib \ --compile-imported-libraries \ --script /tmp/static-precompile-deps.ss # Build static pcre2 shim object RUN PCRE2_CFLAGS=$(pkg-config --cflags libpcre2-8 2>/dev/null || echo "") && \ gcc -c -O2 -o /deps/chez-pcre2/pcre2_shim.o \ - /deps/chez-pcre2/pcre2_shim.c $PCRE2_CFLAGS -Wall + /deps/chez-pcre2/jerboa_pcre2_shim.c $PCRE2_CFLAGS -Wall # Build static jsh FFI shim object RUN gcc -c -O2 -o /deps/jsh/jsh_ffi_shim.o /deps/jsh/ffi-shim.c -Wall --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ VENDOR_REPOS := jerboa jerboa-shell jerboa-pcre2 jerboa-scintilla jerboa-qt jerb ifeq ($(JEMACS_ENABLE_AWS),1) VENDOR_REPOS += jerboa-aws endif +JSH_VENDOR_REPOS := jerboa-coreutils jerboa-awk jerboa-sed jerboa-aws jerboa-yubikey jerboa-ssh jerboa-fuse JERBOA ?= $(VENDOR)/jerboa VENDORED_JERBOA ?= $(JERBOA)/dist/jerboa @@ -50,6 +51,7 @@ JERBOA_FUSE ?= $(JSH_VENDOR)/jerboa-fuse/lib JSQLITE_ROOT ?= $(VENDOR)/jsqlite JSQLITE_DIR ?= $(JSQLITE_ROOT)/src JSQLITE_LIBDIR ?= $(JSH_ROOT)/_jerbuild-stage +JSH_MAKE_ARGS = VENDOR="$(abspath $(JSH_VENDOR))" JSQLITE_REPO="$(abspath $(JSQLITE_ROOT))" JSQLITE_DIR="$(abspath $(JSQLITE_DIR))" JERBOA_WORMHOLE="$(abspath $(JSH_VENDOR)/jerboa-wormhole)" JAWS_ROOT ?= $(VENDOR)/jerboa-aws JAWS ?= $(JAWS_ROOT)/lib JERBOA_SSL_ROOT ?= $(VENDOR)/jerboa-ssl @@ -199,6 +201,28 @@ vendor-deps: git clone --depth 1 "$$url" "$$dest"; \ fi; \ done + @echo "=== Vendoring jsh dependencies into $(JSH_VENDOR) ===" + @mkdir -p $(JSH_VENDOR) + @for repo in $(JSH_VENDOR_REPOS); do \ + dest="$(JSH_VENDOR)/$$repo"; \ + if [ -d "$$dest" ] && [ -n "$$(ls -A "$$dest")" ]; then \ + echo " (exists) $$repo"; \ + elif [ -d "$$dest" ]; then \ + echo " replacing empty directory $$dest"; \ + rmdir "$$dest"; \ + echo " cloning $$repo"; \ + git clone --depth 1 "https://git.sr.ht/~lisp/$$repo" "$$dest"; \ + rm -rf "$$dest/.git"; \ + elif [ -e "$$dest" ]; then \ + echo "ERROR: $$dest exists but is not a directory; remove it and rerun make vendor-deps" >&2; \ + exit 1; \ + else \ + echo " cloning $$repo"; \ + git clone --depth 1 "https://git.sr.ht/~lisp/$$repo" "$$dest"; \ + rm -rf "$$dest/.git"; \ + fi; \ + done + @sh scripts/patch-jerboa-std-compat.sh "$(abspath $(JERBOA)/lib)" @sh scripts/patch-jerboa-shell-compat.sh "$(CURDIR)" @scripts/patch-jerboa-aws-compat.sh "$(CURDIR)" @echo "=== Vendor complete ===" @@ -221,11 +245,10 @@ check-pcre2-system-deps: rm -f "$$tmp.c" "$$tmp" deps: check-pcre2-system-deps vendor-deps ensure-vendor-jerboa-tools - $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" JSQLITE_REPO="$(abspath $(JSQLITE_ROOT))" JSQLITE_DIR="$(abspath $(JSQLITE_DIR))" vendor-deps sh scripts/patch-jerboa-shell-compat.sh "$(CURDIR)" - $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" JSQLITE_REPO="$(abspath $(JSQLITE_ROOT))" JSQLITE_DIR="$(abspath $(JSQLITE_DIR))" stage-jsqlite - $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" JSQLITE_REPO="$(abspath $(JSQLITE_ROOT))" JSQLITE_DIR="$(abspath $(JSQLITE_DIR))" jerboa - $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" JSQLITE_REPO="$(abspath $(JSQLITE_ROOT))" JSQLITE_DIR="$(abspath $(JSQLITE_DIR))" gen-embed </dev/null + $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" $(JSH_MAKE_ARGS) stage-jsqlite + $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" $(JSH_MAKE_ARGS) jerboa + $(MAKE) -C $(JSH_ROOT) JERBUILD="$(VENDORED_JERBUILD)" $(JSH_MAKE_ARGS) gen-embed </dev/null $(MAKE) -C $(JERBOA) build $(MAKE) -C $(JERBOA_PCRE2_ROOT) SCHEME=$(SCHEME) JERBOA_HOME=$(JERBOA) JERBUILD=$(VENDORED_JERBUILD) $(MAKE) -C $(JERBOA_SCINTILLA_ROOT) JERBOA_HOME=$(JERBOA) JERBUILD=$(VENDORED_JERBUILD) transpile @@ -357,8 +380,8 @@ libqt_shim.$(SHLIB_EXT): vendor/qt_shim.cpp -o libqt_shim.$(SHLIB_EXT) \ $(QT_LIBS) $(QSCI_LIBS) -lvterm $(PTY_LINK) -jerboa_qt_shim.$(SHLIB_EXT): vendor/jerboa_qt_shim.c vendor/qt_shim.h - gcc $(SHLIB_FLAGS) -O2 -o jerboa_qt_shim.$(SHLIB_EXT) vendor/jerboa_qt_shim.c -Ivendor -DQT_SCINTILLA_AVAILABLE -Wall +jerboa_qt_shim.$(SHLIB_EXT): vendor/qt_chez_shim.c vendor/qt_shim.h + gcc $(SHLIB_FLAGS) -O2 -o jerboa_qt_shim.$(SHLIB_EXT) vendor/qt_chez_shim.c -Ivendor -DQT_SCINTILLA_AVAILABLE -Wall libjsh-ffi.$(SHLIB_EXT): $(JSH_ROOT)/ffi-shim.c gcc $(SHLIB_FLAGS) -O2 -o libjsh-ffi.$(SHLIB_EXT) $(JSH_ROOT)/ffi-shim.c -Wall @@ -763,7 +786,8 @@ build-jemacs-qt-static: check-root cd /src && find lib -name '*.so' -o -name '*.wpo' | xargs rm -f 2>/dev/null; \ find /src/src/jerboa-emacs -name '*.ss' | sed 's|/src/src/|/src/lib/|; s|\.ss$$|.sls|' | xargs rm -f 2>/dev/null; \ rm -f /src/src/.jerbuild-hashes 2>/dev/null; \ - cd /src && make build SCHEME=/opt/chez/bin/scheme JERBOA=/deps/jerboa && \ + cd /src && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ + --script /deps/jerboa/jerbuild.ss src/ lib/ && \ if [ -f /src/vendor/qt_shim.cpp ]; then \ echo "Rebuilding libqt_shim.a from updated qt_shim.cpp..." && \ QT_CFLAGS=$$(pkg-config --cflags Qt6Widgets 2>/dev/null || \ @@ -777,17 +801,32 @@ build-jemacs-qt-static: check-root ar rcs /deps/qt-shim/libqt_shim.a \ /deps/qt-shim/qt_shim_static.o; \ fi && \ - cp -a /src/vendor/jerboa-qt/. /deps/jerboa-qt/ && \ + rm -rf /deps/jerboa-qt && mkdir -p /deps/jerboa-qt/jerboa-qt /deps/jerboa-qt/src && \ + cp /src/vendor/jerboa-qt/lib/jerboa-qt/*.sls /deps/jerboa-qt/jerboa-qt/ && \ find /deps/jerboa-qt -name '*.so' -delete && \ find /deps/jerboa-qt -name '*.wpo' -delete && \ - /opt/chez/bin/scheme --libdirs /deps/jerboa-qt \ - --compile-imported-libraries --script /deps/jerboa-qt/compile-libs.ss && \ - cp /src/vendor/jerboa-pcre2-ffi-static.ss /deps/jerboa-pcre2/jerboa-pcre2/ffi.ss && \ - /opt/chez/bin/scheme --libdirs /deps/jerboa-pcre2 \ - --compile-imported-libraries --script /src/vendor/jerboa-pcre2-compile-libs.ss && \ - cp /src/vendor/jerboa-scintilla-ffi-static.sls /deps/jerboa-scintilla/src/jerboa-scintilla/ffi.sls && \ - JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa-scintilla/src \ - --compile-imported-libraries --script /src/vendor/jerboa-scintilla-compile-libs.ss && \ + printf '%s\n' '(import (jerboa-qt ffi) (jerboa-qt qt))' > /tmp/compile-jerboa-qt.ss && \ + JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa-qt:/deps/jerboa/lib \ + --compile-imported-libraries --script /tmp/compile-jerboa-qt.ss && \ + rm -rf /deps/jerboa-pcre2 && mkdir -p /deps/jerboa-pcre2 && \ + cp /src/vendor/jerboa-pcre2/jerboa_pcre2_shim.c /deps/jerboa-pcre2/pcre2_shim.c && \ + PCRE2_CFLAGS=$$(pkg-config --cflags libpcre2-8 2>/dev/null || echo "") && \ + gcc -shared -fPIC -O2 -o /deps/jerboa-pcre2/jerboa_pcre2_shim.so \ + /deps/jerboa-pcre2/pcre2_shim.c $$PCRE2_CFLAGS -lpcre2-8 && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ + --script /deps/jerboa/jerbuild.ss /src/vendor/jerboa-pcre2/src /deps/jerboa-pcre2 --force && \ + perl -0pi -e 's/\(def shim-loaded\s+\(load-shared-object\s+\(let \(\[env \(getenv "JERBOA_PCRE2_LIB"\)\]\)\s+\(if env\s+\(string-append env "\/jerboa_pcre2_shim\." shlib-ext\)\s+\(string-append "jerboa_pcre2_shim\." shlib-ext\)\)\)\)\)/(def shim-loaded\n (if (foreign-entry? "jerboa_pcre2_compile")\n #t\n (load-shared-object\n (let ([env (getenv "JERBOA_PCRE2_LIB")])\n (if env\n (string-append env "\/jerboa_pcre2_shim." shlib-ext)\n (string-append "jerboa_pcre2_shim." shlib-ext))))))/s' \ + /deps/jerboa-pcre2/jerboa-pcre2/ffi.sls && \ + printf '%s\n' '(import (jerboa-pcre2 pcre2))' > /tmp/compile-jerboa-pcre2.ss && \ + JERBOA_PCRE2_LIB=/deps/jerboa-pcre2 /opt/chez/bin/scheme --libdirs /deps/jerboa-pcre2:/deps/jerboa/lib \ + --compile-imported-libraries --script /tmp/compile-jerboa-pcre2.ss && \ + rm -rf /deps/jerboa-scintilla && mkdir -p /deps/jerboa-scintilla/src/jerboa-scintilla && \ + cp /src/vendor/jerboa-scintilla/lib/jerboa-scintilla/*.sls /deps/jerboa-scintilla/src/jerboa-scintilla/ && \ + find /deps/jerboa-scintilla -name '*.so' -delete && \ + find /deps/jerboa-scintilla -name '*.wpo' -delete && \ + printf '%s\n' '(import (jerboa-scintilla constants) (jerboa-scintilla scintilla) (jerboa-scintilla ffi) (jerboa-scintilla style) (jerboa-scintilla tui))' > /tmp/compile-jerboa-scintilla.ss && \ + JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa-scintilla/src:/deps/jerboa/lib \ + --compile-imported-libraries --script /tmp/compile-jerboa-scintilla.ss && \ cp /src/vendor/jerboa-net-tcp-static.sls /deps/jerboa/lib/std/net/tcp.sls && \ cp /src/vendor/jerboa-net-tcp-raw-static.sls /deps/jerboa/lib/std/net/tcp-raw.sls && \ cp /src/vendor/jerboa-net-uri.sls /deps/jerboa/lib/std/net/uri.sls && \ @@ -811,7 +850,7 @@ build-jemacs-qt-static: check-root --compile-imported-libraries --script /src/vendor/jerboa-compile-repl.ss && \ cd /src && \ rm -f /src/lib/jerboa/*.wpo /src/lib/jerboa/*.so && \ - JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib \ + JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib:/deps/jerboa/lib \ --compile-imported-libraries --script /src/vendor/jerboa-compile-repl-socket.ss && \ echo "Building tree-sitter shim + queries..." && \ gcc -c -O2 -I/opt/tree-sitter-include -o /tmp/jemacs-build/treesitter_shim.o \ @@ -854,17 +893,17 @@ build-jemacs-qt-static: check-root TREE_SITTER_QUERIES_OBJ=/tmp/jemacs-build/treesitter_queries.o \ PKG_CONFIG_PATH=/opt/qt6-static/lib/pkgconfig \ /opt/chez/bin/scheme \ - --libdirs lib:/deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/jsh/vendor/jerboa-ssh/src:/deps/jsh/vendor/jerboa-sqlite/src:/deps/jsh/vendor/jerboa-crypto/src:/deps/jsh/vendor/jerboa-fuse/lib:/deps/jerboa-pcre2:/deps/jerboa-scintilla/src:/deps/jerboa-qt$(AWS_STATIC_LIBDIRS) \ + --libdirs lib:/deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jsqlite/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/jsh/vendor/jerboa-ssh/src:/deps/jsh/vendor/jerboa-sqlite/src:/deps/jsh/vendor/jerboa-crypto/src:/deps/jsh/vendor/jerboa-fuse/lib:/deps/jerboa-pcre2:/deps/jerboa-scintilla/src:/deps/jerboa-qt$(AWS_STATIC_LIBDIRS) \ --script build-binary-qt.ss -linux-static-qt-podman: ensure-podman-deps +linux-static-qt-podman: vendor-deps ensure-podman-deps $(PODMAN) run --rm $(PODMAN_RUN_FLAGS) \ --ulimit nofile=8192:8192 \ -v $(CURDIR):/src:z \ + -v $(JSQLITE_ROOT):/host-jsqlite:ro \ -v $(JERBOA)/lib/std:/host-jerboa-std:ro \ -v $(JERBOA)/lib/jerboa:/host-jerboa-core:ro \ - -v $(JSH_SRC)/src:/host-jsh-src:ro \ - -v $(JSH_VENDOR_SRC):/host-jsh-vendor:ro \ + -v $(JSH_SRC):/host-jsh-root:ro \ -v $(JSH_COREUTILS_LIB):/host-jsh-coreutils.a:ro \ $(AWS_PODMAN_RUN_MOUNTS) \ $(DEPS_IMAGE) \ @@ -878,11 +917,13 @@ linux-static-qt-podman: ensure-podman-deps cd / && rm -rf /tmp/openssl-3.3.2*; \ fi; \ cp /host-jsh-coreutils.a /deps/jsh/libjsh_coreutils.a; \ - cp -a /host-jsh-src/. /deps/jsh/src/; \ - mkdir -p /deps/jsh/vendor && cp -a /host-jsh-vendor/. /deps/jsh/vendor/; \ + cp -a /host-jsh-root/. /deps/jsh/; \ + mkdir -p /deps/jsh/vendor/jsqlite && cp -a /host-jsqlite/. /deps/jsh/vendor/jsqlite/; \ + sh /src/scripts/patch-jsqlite-compat.sh /deps/jsh/vendor/jsqlite/src/jsqlite/json.ss; \ echo 'SYNC: bulk-copying host jerboa std/ and jerboa/ into container...'; \ cp -a /host-jerboa-std/. /deps/jerboa/lib/std/ && \ cp -a /host-jerboa-core/. /deps/jerboa/lib/jerboa/ && \ + sh /src/scripts/patch-jerboa-std-compat.sh /deps/jerboa/lib && \ find /deps/jerboa/lib -name '*.so' -delete && \ find /deps/jerboa/lib -name '*.wpo' -delete && \ echo '(import (chezscheme)) (compile-imported-libraries #t) (generate-wpo-files #t) (import (jerboa core)) (import (jerboa prelude))' \ @@ -890,6 +931,28 @@ linux-static-qt-podman: ensure-podman-deps cd /deps/jerboa/lib && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ -q --script /tmp/compile-jerboa-core.ss && \ echo 'COMPILED: jerboa core + prelude'; \ + echo 'GENERATE: jsh libraries from vendored sources'; \ + mkdir -p /deps/jsh/src; \ + find /deps/jsh/src \( -name '*.so' -o -name '*.wpo' \) -delete 2>/dev/null; \ + rm -rf /deps/jsh/jsh-src && mkdir -p /deps/jsh/jsh-src/jsh; \ + cd /deps/jsh && for m in ast registry macros pregexp-compat util environment lexer arithmetic glob fuzzy history recorder player parser functions signals expander redirect control jobs builtins pipeline executor completion prompt procwatch lineedit fzf script startup main; do \ + if [ -f \"\$$m.ss\" ]; then \ + cp \"\$$m.ss\" \"jsh-src/jsh/\$$m.ss\"; \ + elif [ -f \"jerboa-shell/\$$m.ss\" ]; then \ + cp \"jerboa-shell/\$$m.ss\" \"jsh-src/jsh/\$$m.ss\"; \ + fi; \ + done && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib --script support/jerbuild.ss jsh-src src --force && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib --script support/jerbuild.ss jerboa-src/src src --force && \ + printf '%s\n' '#!chezscheme' '(library (jsh embed-data)' \ + ' (export %embed-file-table %embed-encrypted? %embed-salt %record-pubkey)' \ + ' (import (chezscheme))' \ + ' (define %embed-encrypted? #f)' \ + ' (define %embed-salt #vu8())' \ + ' (define %record-pubkey #f)' \ + ' (define %embed-file-table (make-hashtable string-hash string=?))' \ + ')' > /deps/jsh/src/jsh/embed-data.sls && \ + echo 'GENERATED: jsh libraries'; \ if [ '$(JEMACS_ENABLE_AWS)' = '1' ]; then \ mkdir -p /deps/jerboa-aws /deps/jerboa-ssl/src /deps/jerboa-https/src && \ cp -a /host-jaws/. /deps/jerboa-aws/ && \ @@ -929,7 +992,8 @@ build-jemacs-tui-static: check-root fi; \ cd /src && find lib -name '*.so' -o -name '*.wpo' | xargs rm -f 2>/dev/null; \ find /src/src/jerboa-emacs -name '*.ss' | sed 's|/src/src/|/src/lib/|; s|\.ss$$|.sls|' | xargs rm -f 2>/dev/null; \ - cd /src && make build SCHEME=/opt/chez/bin/scheme JERBOA=/deps/jerboa && \ + cd /src && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ + --script /deps/jerboa/jerbuild.ss src/ lib/ && \ cp /src/vendor/jerboa-pcre2-ffi-static.ss /deps/jerboa-pcre2/jerboa-pcre2/ffi.ss && \ /opt/chez/bin/scheme --libdirs /deps/jerboa-pcre2 \ --compile-imported-libraries --script /src/vendor/jerboa-pcre2-compile-libs.ss && \ @@ -959,7 +1023,7 @@ build-jemacs-tui-static: check-root --compile-imported-libraries --script /src/vendor/jerboa-compile-repl.ss && \ cd /src && \ rm -f /src/lib/jerboa/*.wpo /src/lib/jerboa/*.so && \ - JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib \ + JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib:/deps/jerboa/lib \ --compile-imported-libraries --script /src/vendor/jerboa-compile-repl-socket.ss && \ echo "Building Scintilla+termbox+Lexilla static archives from source..." && \ make -C /deps/sci-vendor/scintilla/termbox/termbox_next && \ @@ -974,29 +1038,31 @@ build-jemacs-tui-static: check-root SCI_VENDOR_DIR=/deps/sci-vendor \ JSH_COREUTILS_LIB=/deps/jsh/libjsh_coreutils.a \ /opt/chez/bin/scheme \ - --libdirs lib:/deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/jsh/vendor/jerboa-ssh/src:/deps/jsh/vendor/jerboa-sqlite/src:/deps/jsh/vendor/jerboa-crypto/src:/deps/jsh/vendor/jerboa-fuse/lib:/deps/jerboa-pcre2:/deps/jerboa-scintilla/src \ + --libdirs lib:/deps/jerboa/lib:/deps/jsh/src:/deps/jsh/vendor/jsqlite/src:/deps/jsh/vendor/jerboa-coreutils/lib:/deps/jsh/vendor/jerboa-awk/lib:/deps/jsh/vendor/jerboa-sed/lib:/deps/jsh/vendor/jerboa-ssh/src:/deps/jsh/vendor/jerboa-sqlite/src:/deps/jsh/vendor/jerboa-crypto/src:/deps/jsh/vendor/jerboa-fuse/lib:/deps/jerboa-pcre2:/deps/jerboa-scintilla/src \ --script build-binary.ss -linux-static-tui-podman: ensure-podman-deps +linux-static-tui-podman: vendor-deps ensure-podman-deps @test -d $(SCI_VENDOR_SRC) || \ { echo "ERROR: SCI_VENDOR_SRC='$(SCI_VENDOR_SRC)' not found. Populate it from the jerboa-scintilla dependency or set SCI_VENDOR_SRC."; exit 1; } $(PODMAN) run --rm $(PODMAN_RUN_FLAGS) \ --ulimit nofile=8192:8192 \ -v $(CURDIR):/src:z \ + -v $(JSQLITE_ROOT):/host-jsqlite:ro \ -v $(SCI_VENDOR_SRC):/deps/sci-vendor:ro \ -v $(JERBOA)/lib/std:/host-jerboa-std:ro \ -v $(JERBOA)/lib/jerboa:/host-jerboa-core:ro \ - -v $(JSH_SRC)/src:/host-jsh-src:ro \ - -v $(JSH_VENDOR_SRC):/host-jsh-vendor:ro \ + -v $(JSH_SRC):/host-jsh-root:ro \ -v $(JSH_COREUTILS_LIB):/host-jsh-coreutils.a:ro \ $(DEPS_IMAGE) \ sh -c "apk add --no-cache libvterm-dev libvterm-static; \ cp /host-jsh-coreutils.a /deps/jsh/libjsh_coreutils.a; \ - cp -a /host-jsh-src/. /deps/jsh/src/; \ - mkdir -p /deps/jsh/vendor && cp -a /host-jsh-vendor/. /deps/jsh/vendor/; \ + cp -a /host-jsh-root/. /deps/jsh/; \ + mkdir -p /deps/jsh/vendor/jsqlite && cp -a /host-jsqlite/. /deps/jsh/vendor/jsqlite/; \ + sh /src/scripts/patch-jsqlite-compat.sh /deps/jsh/vendor/jsqlite/src/jsqlite/json.ss; \ echo 'SYNC: bulk-copying host jerboa std/ and jerboa/ into container...'; \ cp -a /host-jerboa-std/. /deps/jerboa/lib/std/ && \ cp -a /host-jerboa-core/. /deps/jerboa/lib/jerboa/ && \ + sh /src/scripts/patch-jerboa-std-compat.sh /deps/jerboa/lib && \ find /deps/jerboa/lib -name '*.so' -delete && \ find /deps/jerboa/lib -name '*.wpo' -delete && \ echo '(import (chezscheme)) (compile-imported-libraries #t) (generate-wpo-files #t) (import (jerboa core)) (import (jerboa prelude))' \ @@ -1004,6 +1070,28 @@ linux-static-tui-podman: ensure-podman-deps cd /deps/jerboa/lib && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ -q --script /tmp/compile-jerboa-core.ss && \ echo 'COMPILED: jerboa core + prelude'; \ + echo 'GENERATE: jsh libraries from vendored sources'; \ + mkdir -p /deps/jsh/src; \ + find /deps/jsh/src \( -name '*.so' -o -name '*.wpo' \) -delete 2>/dev/null; \ + rm -rf /deps/jsh/jsh-src && mkdir -p /deps/jsh/jsh-src/jsh; \ + cd /deps/jsh && for m in ast registry macros pregexp-compat util environment lexer arithmetic glob fuzzy history recorder player parser functions signals expander redirect control jobs builtins pipeline executor completion prompt procwatch lineedit fzf script startup main; do \ + if [ -f \"\$$m.ss\" ]; then \ + cp \"\$$m.ss\" \"jsh-src/jsh/\$$m.ss\"; \ + elif [ -f \"jerboa-shell/\$$m.ss\" ]; then \ + cp \"jerboa-shell/\$$m.ss\" \"jsh-src/jsh/\$$m.ss\"; \ + fi; \ + done && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib --script support/jerbuild.ss jsh-src src --force && \ + /opt/chez/bin/scheme --libdirs /deps/jerboa/lib --script support/jerbuild.ss jerboa-src/src src --force && \ + printf '%s\n' '#!chezscheme' '(library (jsh embed-data)' \ + ' (export %embed-file-table %embed-encrypted? %embed-salt %record-pubkey)' \ + ' (import (chezscheme))' \ + ' (define %embed-encrypted? #f)' \ + ' (define %embed-salt #vu8())' \ + ' (define %record-pubkey #f)' \ + ' (define %embed-file-table (make-hashtable string-hash string=?))' \ + ')' > /deps/jsh/src/jsh/embed-data.sls && \ + echo 'GENERATED: jsh libraries'; \ mkdir -p /tmp/jemacs-build && \ exec env HOME=/tmp/jemacs-build sh -c '\ cd /src && make build-jemacs-tui-static'" --- a/build-binary-qt.ss +++ b/build-binary-qt.ss @@ -312,6 +312,13 @@ (if (eof-object? line) default line)) default))) +(define (first-existing-file paths) + (let loop ((rest paths)) + (cond + ((null? rest) #f) + ((file-exists? (car rest)) (car rest)) + (else (loop (cdr rest)))))) + (when jemacs-static? (printf "Static build mode: JEMACS_STATIC=1~n")) @@ -329,7 +336,7 @@ ;; jerboa_qt_shim — use jerboa-emacs deferred-callback version (avoids Sactivate_thread GC deadlock) (let* ((qt-cflags (shell-output "pkg-config --cflags Qt6Widgets" ""))) ;; Static .o for static builds - (let ((cmd (format "gcc -c -O2 -DQT_SCINTILLA_AVAILABLE -o jemacs-qt-chez-shim.o vendor/jerboa_qt_shim.c -I~a ~a -Wall 2>&1" + (let ((cmd (format "gcc -c -O2 -DQT_SCINTILLA_AVAILABLE -o jemacs-qt-chez-shim.o vendor/qt_chez_shim.c -I~a ~a -Wall 2>&1" qt-shim-dir qt-cflags))) (unless (= 0 (system cmd)) (display "Error: jerboa_qt_shim compilation failed\n") @@ -339,7 +346,7 @@ (let* ((shlib-flags (if macos-build? "-dynamiclib -Wl,-undefined,dynamic_lookup" "-shared -fPIC")) - (cmd (format "gcc ~a -O2 -DQT_SCINTILLA_AVAILABLE -o jerboa_qt_shim.~a vendor/jerboa_qt_shim.c -I~a ~a -Wall 2>&1" + (cmd (format "gcc ~a -O2 -DQT_SCINTILLA_AVAILABLE -o jerboa_qt_shim.~a vendor/qt_chez_shim.c -I~a ~a -Wall 2>&1" shlib-flags shlib-ext qt-shim-dir qt-cflags))) (unless (= 0 (system cmd)) (display "Error: jerboa_qt_shim shlib compilation failed\n") @@ -349,8 +356,14 @@ ;; (dlopen(NULL) is a stub in musl static builds, so we use Sforeign_symbol instead) (when jemacs-static? ;; scheme.h lives in chez-dir (same dir as libkernel.a) - (let* ((ffi-path (format "~a/jerboa-qt/ffi.ss" qt-dir)) - (pcre2-ffi-path (format "~a/jerboa-pcre2/ffi.ss" pcre2-dir)) + (let* ((ffi-path + (first-existing-file + (list (format "~a/jerboa-qt/ffi.ss" qt-dir) + (format "~a/jerboa-qt/ffi.sls" qt-dir)))) + (pcre2-ffi-path + (first-existing-file + (list (format "~a/jerboa-pcre2/ffi.ss" pcre2-dir) + (format "~a/jerboa-pcre2/ffi.sls" pcre2-dir)))) (include-dir chez-dir) ;; Generate the C file by scanning ALL FFI source files: ;; jerboa-qt/ffi.ss, jerboa-pcre2/ffi.ss — specific files @@ -372,7 +385,7 @@ find ~a -name '*.sls' -o -name '*.ss' | \ sed 's/.*define-foreign [^ ]* *\"//;s/\".*//'; } | \ sort -u | grep -v '^$' | grep -E '^[A-Za-z_][A-Za-z0-9_]*$' | grep -v '^_NSGetExecutablePath$' | grep -v '^io_uring_' | \ grep -v '^jerboa_' | grep -v '^SSL_' | grep -v '^TLS_' | grep -v '^EVP_' | \ -grep -v '^CRYPTO_' | grep -v '^PKCS5_' | grep -v '^RAND_' | \ +grep -v '^CRYPTO_' | grep -v '^PKCS5_' | grep -v '^RAND_' | grep -v '^HMAC$' | \ grep -v '^QRcode_' | grep -v '^embed_encrypt$' | grep -v '^embed_random_bytes$' | \ grep -v '^kqueue$' | grep -v '^kevent$' | grep -v '^sandbox_' | \ grep -v '^__error$' | grep -v '^__errno$' | grep -v '^cap_' | \ @@ -393,6 +406,12 @@ echo OK" ffi-path pcre2-ffi-path jsh-dir)) (result (shell-output gen-cmd ""))) + (unless ffi-path + (printf "Error: jerboa-qt ffi source/library not found in ~a~n" qt-dir) + (exit 1)) + (unless pcre2-ffi-path + (printf "Error: jerboa-pcre2 ffi source/library not found in ~a~n" pcre2-dir) + (exit 1)) (unless (string=? (string-downcase (substring result 0 (min 2 (string-length result)))) "ok") (printf "Error generating qt_static_symbols.c: ~a~n" result) (exit 1)) @@ -466,9 +485,9 @@ echo OK" ;; jerboa-scintilla stubs (TUI-only — Qt never calls these; stubs allow foreign-procedure defs) (when jemacs-static? - (let* ((cmd "gcc -c -O2 -o jemacs-qt-sci-stubs.o support/jerboa_scintilla_stubs.c -Wall 2>&1")) + (let* ((cmd "gcc -c -O2 -o jemacs-qt-sci-stubs.o support/chez_scintilla_stubs.c -Wall 2>&1")) (unless (= 0 (system cmd)) - (display "Error: jerboa_scintilla_stubs.c compilation failed\n") + (display "Error: chez_scintilla_stubs.c compilation failed\n") (exit 1)))) ;; jerboa-ssl shim (TLS FFI for jerboa-aws EC2 API calls) new file mode 100644 --- /dev/null +++ b/scripts/patch-jerboa-std-compat.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +libdir=${1:?usage: patch-jerboa-std-compat.sh JERBOA_LIBDIR} + +patch_import() { + file=$1 + + [ -f "$file" ] || return 0 + if grep -q "^ (only (jerboa core) def defstruct))$" "$file"; then + return 0 + fi + + tmp="${file}.tmp.$$" + awk ' + $0 == " (import (chezscheme))" { + print " (import (chezscheme)" + print " (only (jerboa core) def defstruct))" + next + } + { print } + ' "$file" > "$tmp" + mv "$tmp" "$file" + + if ! grep -q "^ (only (jerboa core) def defstruct))$" "$file"; then + echo "patch-jerboa-std-compat: failed to patch $file" >&2 + exit 1 + fi +} + +patch_import "$libdir/std/misc/lru-cache.ss" +patch_import "$libdir/std/misc/rbtree.ss" new file mode 100644 --- /dev/null +++ b/scripts/patch-jsqlite-compat.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +file=${1:?usage: patch-jsqlite-compat.sh path/to/jsqlite/json.ss} + +[ -f "$file" ] || { + echo "ERROR: jsqlite JSON source not found: $file" >&2 + exit 1 +} + +if grep -q 'jsqlite-bytevector-append' "$file"; then + exit 0 +fi + +tmp="${file}.tmp.$$" +awk ' + /^ ;; ---- JSONB binary codec/ && !inserted { + print " (def (jsqlite-bytevector-append . bvs)" + print " (let ([total (let loop ([xs bvs] [n 0])" + print " (if (null? xs)" + print " n" + print " (loop (cdr xs) (+ n (bytevector-length (car xs))))))])" + print " (let ([out (make-bytevector total)])" + print " (let loop ([xs bvs] [pos 0])" + print " (if (null? xs)" + print " out" + print " (let* ([bv (car xs)] [len (bytevector-length bv)])" + print " (let copy ([i 0])" + print " (when (< i len)" + print " (bytevector-u8-set! out (+ pos i) (bytevector-u8-ref bv i))" + print " (copy (+ i 1))))" + print " (loop (cdr xs) (+ pos len))))))))" + print "" + inserted = 1 + } + { + gsub(/\(bytevector-append /, "(jsqlite-bytevector-append ") + gsub(/\(apply bytevector-append/, "(apply jsqlite-bytevector-append") + print + } + END { + if (!inserted) { + exit 2 + } + } +' "$file" > "$tmp" || { + rc=$? + rm -f "$tmp" + exit "$rc" +} +mv "$tmp" "$file" --- a/src/jerboa-emacs/chez-powers.ss +++ b/src/jerboa-emacs/chez-powers.ss @@ -54,7 +54,6 @@ (import :std/sugar :std/stm :std/engine - :std/misc/lru-cache :std/misc/wg :std/misc/channel :jerboa-emacs/atom @@ -64,3 +63,117 @@ :std/misc/completion :std/misc/barrier :std/amb) + +;; Local LRU cache implementation. The vendored std/misc/lru-cache module uses +;; Jerboa forms without importing their macros, so static builds cannot compile it. +(defstruct node-rec-raw (key value prev next)) +(def (make-node-rec key value) (make-node-rec-raw key value #f #f)) +(def node-rec-key node-rec-raw-key) +(def node-rec-value node-rec-raw-value) +(def node-rec-value-set! node-rec-raw-value-set!) +(def node-rec-prev node-rec-raw-prev) +(def node-rec-prev-set! node-rec-raw-prev-set!) +(def node-rec-next node-rec-raw-next) +(def node-rec-next-set! node-rec-raw-next-set!) + +(defstruct lru-cache-rec-raw (capacity table head tail size hits misses)) +(def (make-lru-cache-rec cap) + (make-lru-cache-rec-raw cap (make-hashtable equal-hash equal?) #f #f 0 0 0)) +(def lru-cache? lru-cache-rec-raw?) +(def lru-cache-capacity lru-cache-rec-raw-capacity) +(def lru-cache-rec-table lru-cache-rec-raw-table) +(def lru-cache-rec-head lru-cache-rec-raw-head) +(def lru-cache-rec-head-set! lru-cache-rec-raw-head-set!) +(def lru-cache-rec-tail lru-cache-rec-raw-tail) +(def lru-cache-rec-tail-set! lru-cache-rec-raw-tail-set!) +(def lru-cache-size lru-cache-rec-raw-size) +(def lru-cache-rec-size-set! lru-cache-rec-raw-size-set!) +(def lru-cache-rec-hits lru-cache-rec-raw-hits) +(def lru-cache-rec-hits-set! lru-cache-rec-raw-hits-set!) +(def lru-cache-rec-misses lru-cache-rec-raw-misses) +(def lru-cache-rec-misses-set! lru-cache-rec-raw-misses-set!) + +(def (make-lru-cache cap) + (unless (> cap 0) (error 'make-lru-cache "capacity must be positive" cap)) + (make-lru-cache-rec cap)) + +(def lru-cache-get + (case-lambda + ((c key) (lru-cache-get c key #f)) + ((c key default) + (let ((node (hashtable-ref (lru-cache-rec-table c) key #f))) + (if node + (begin + (lru-cache-rec-hits-set! c (+ (lru-cache-rec-hits c) 1)) + (lru-cache-move-to-head! c node) + (node-rec-value node)) + (begin + (lru-cache-rec-misses-set! c (+ (lru-cache-rec-misses c) 1)) + default)))))) + +(def (lru-cache-put! c key value) + (let ((existing (hashtable-ref (lru-cache-rec-table c) key #f))) + (if existing + (begin + (node-rec-value-set! existing value) + (lru-cache-move-to-head! c existing)) + (begin + (when (= (lru-cache-size c) (lru-cache-capacity c)) + (lru-cache-evict-tail! c)) + (let ((node (make-node-rec key value))) + (hashtable-set! (lru-cache-rec-table c) key node) + (lru-cache-rec-size-set! c (+ (lru-cache-size c) 1)) + (lru-cache-add-to-head! c node)))))) + +(def (lru-cache-delete! c key) + (let ((node (hashtable-ref (lru-cache-rec-table c) key #f))) + (when node + (lru-cache-remove-node! c node) + (hashtable-delete! (lru-cache-rec-table c) key) + (lru-cache-rec-size-set! c (- (lru-cache-size c) 1))))) + +(def (lru-cache-clear! c) + (hashtable-clear! (lru-cache-rec-table c)) + (lru-cache-rec-head-set! c #f) + (lru-cache-rec-tail-set! c #f) + (lru-cache-rec-size-set! c 0)) + +(def (lru-cache-stats c) + `((size . ,(lru-cache-size c)) + (capacity . ,(lru-cache-capacity c)) + (hits . ,(lru-cache-rec-hits c)) + (misses . ,(lru-cache-rec-misses c)) + (hit-rate . ,(let ((total (+ (lru-cache-rec-hits c) (lru-cache-rec-misses c)))) + (if (= total 0) 0.0 + (inexact (/ (lru-cache-rec-hits c) total))))))) + +(def (lru-cache-add-to-head! c node) + (node-rec-prev-set! node #f) + (node-rec-next-set! node (lru-cache-rec-head c)) + (when (lru-cache-rec-head c) + (node-rec-prev-set! (lru-cache-rec-head c) node)) + (lru-cache-rec-head-set! c node) + (unless (lru-cache-rec-tail c) + (lru-cache-rec-tail-set! c node))) + +(def (lru-cache-remove-node! c node) + (let ((prev (node-rec-prev node)) + (next (node-rec-next node))) + (if prev + (node-rec-next-set! prev next) + (lru-cache-rec-head-set! c next)) + (if next + (node-rec-prev-set! next prev) + (lru-cache-rec-tail-set! c prev)))) + +(def (lru-cache-move-to-head! c node) + (unless (eq? node (lru-cache-rec-head c)) + (lru-cache-remove-node! c node) + (lru-cache-add-to-head! c node))) + +(def (lru-cache-evict-tail! c) + (let ((tail (lru-cache-rec-tail c))) + (when tail + (hashtable-delete! (lru-cache-rec-table c) (node-rec-key tail)) + (lru-cache-remove-node! c tail) + (lru-cache-rec-size-set! c (- (lru-cache-size c) 1))))) --- a/src/jerboa-emacs/qt/sci-shim.ss +++ b/src/jerboa-emacs/qt/sci-shim.ss @@ -256,6 +256,18 @@ (def QT_CURSOR_PREVIOUS_CHAR 5) (def QT_CURSOR_PREVIOUS_WORD 18) +(def ffi-qt-stacked-widget-set-current-widget + (foreign-procedure "qt_stacked_widget_set_current_widget" (void* void*) void)) + +(def (qt-stacked-widget-set-current-widget! sw widget) + (ffi-qt-stacked-widget-set-current-widget sw widget)) + +(def ffi-qt-last-key-widget + (foreign-procedure "qt_last_key_widget" () void*)) + +(def (qt-last-key-widget) + (ffi-qt-last-key-widget)) + ;; Helpers (def (sci-check-handle! caller sci) (unless sci