Keep vendor as build-time checkouts
ober
02294abf3d8c03530a8f30dbadf4a710deb8cf1d
--- a/.gitignore +++ b/.gitignore @@ -38,21 +38,7 @@ pcre2_shim.so qt_chez_shim.so #7778# -# Vendored repos (cloned, not submodules) -vendor/jerboa/ -vendor/jerboa-shell/ -vendor/jerboa-pcre2/ -vendor/jerboa-scintilla/ -vendor/jerboa-qt/ -vendor/jerboa-ssl/ -vendor/jerboa-https/ -vendor/jsqlite/ -vendor/jerboa-lsp/ -vendor/jerboa-aws/ -vendor/chez-pcre2/ -vendor/chez-scintilla/ -vendor/chez-qt/ -vendor/chez-ssl/ -vendor/chez-https/ +# Vendored repos and generated dependency trees are cloned at build time. +/vendor/ vendor/**/*.so vendor/**/*.wpo --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ The editor has automated stress testing infrastructure for finding segfaults. Se ### Crash Reporter -`vendor/qt_shim.cpp` includes a SIGSEGV/SIGBUS/SIGABRT crash reporter that: +`support/vendor-overrides/qt_shim.cpp` includes a SIGSEGV/SIGBUS/SIGABRT crash reporter that: - Records the last 64 FFI calls in a lock-free ring buffer - On crash, writes a diagnostic report to `~/.jemacs-crash.log` with signal details and the FFI ring buffer - Re-raises the signal so gdb/core dumps still work --- a/Dockerfile +++ b/Dockerfile @@ -264,9 +264,9 @@ COPY --from=pcre2-src . /deps/chez-pcre2 COPY --from=sci-src . /deps/chez-scintilla COPY --from=qt-src . /deps/chez-qt RUN mkdir -p /deps/qt-shim -COPY vendor/qt_shim.cpp vendor/qt_shim.h /deps/qt-shim/ -# qt_chez_shim.c is maintained in vendor/ (not in chez-qt) — copy it in -COPY vendor/qt_chez_shim.c vendor/qt_shim.h /deps/chez-qt/ +COPY support/vendor-overrides/qt_shim.cpp support/vendor-overrides/qt_shim.h /deps/qt-shim/ +# qt_chez_shim.c is maintained in support/vendor-overrides/ (not in chez-qt). +COPY support/vendor-overrides/qt_chez_shim.c support/vendor-overrides/qt_shim.h /deps/chez-qt/ # jerboa-shell keeps these support libraries as jerbuild input files. # Transpile only the libraries imported by support/static-precompile-deps.ss. --- a/Dockerfile.qt +++ b/Dockerfile.qt @@ -220,7 +220,7 @@ RUN mkdir -p /opt/tree-sitter-grammars && \ # Separate stage so Qt6 cache is preserved when qt_shim.cpp changes FROM qt-alpine AS qt-shim-alpine -COPY vendor/qt_shim.cpp vendor/qt_shim.h /tmp/qt-shim-src/ +COPY support/vendor-overrides/qt_shim.cpp support/vendor-overrides/qt_shim.h /tmp/qt-shim-src/ RUN mkdir -p /tmp/qt-shim-out && \ QT_CFLAGS="-I/opt/qt6-static/include \ --- a/Makefile +++ b/Makefile @@ -15,7 +15,9 @@ JH = $(shell "$(JERBUILD_TOOL)" --jerboa-home 2>/dev/null) SSH ?= ssh SCP ?= scp LINUX_BUILD_HOST ?= linux.cons.io -LINUX_BUILD_ROOT ?= ~/mine/jerboa-emacs +REMOTE_REPO_URL ?= $(shell git remote get-url origin 2>/dev/null || echo git@git.sr.ht:~lisp/jerboa-emacs) +REMOTE_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo master) +LINUX_BUILD_ROOT ?= .cache/jemacs-build/jerboa-emacs LINUX_ARM64_BUILD_HOST ?= LINUX_ARM64_BUILD_ROOT ?= $(LINUX_BUILD_ROOT) @@ -232,21 +234,19 @@ vendor-deps: @mkdir -p $(JSH_VENDOR) @for repo in $(JSH_VENDOR_REPOS); do \ dest="$(JSH_VENDOR)/$$repo"; \ - if [ -d "$$dest" ] && [ -n "$$(ls -A "$$dest")" ]; then \ + if [ -d "$$dest/.git" ]; then \ echo " (exists) $$repo"; \ - elif [ -d "$$dest" ]; then \ + elif [ -d "$$dest" ] && [ -z "$$(ls -A "$$dest")" ]; then \ echo " replacing empty directory $$dest"; \ rmdir "$$dest"; \ echo " cloning $$repo"; \ git clone --depth 1 "https://git.sr.ht/~lisp/$$repo" "$$dest"; \ - rm -rf "$$dest/.git"; \ elif [ -e "$$dest" ]; then \ - echo "ERROR: $$dest exists but is not a directory; remove it and rerun make vendor-deps" >&2; \ + echo "ERROR: $$dest exists but is not a git clone; remove it and rerun make vendor-deps" >&2; \ exit 1; \ else \ echo " cloning $$repo"; \ git clone --depth 1 "https://git.sr.ht/~lisp/$$repo" "$$dest"; \ - rm -rf "$$dest/.git"; \ fi; \ done @sh scripts/patch-jerboa-std-compat.sh "$(abspath $(JERBOA)/lib)" @@ -461,17 +461,17 @@ QSCI_CFLAGS := QSCI_LIBS := -lqscintilla2_qt6 endif -libqt_shim.$(SHLIB_EXT): vendor/qt_shim.cpp +libqt_shim.$(SHLIB_EXT): support/vendor-overrides/qt_shim.cpp g++ $(SHLIB_FLAGS) -std=c++17 -O2 \ -DJEMACS_CHEZ_SMP -DQT_SCINTILLA_AVAILABLE \ -I$(QT_SHIM_H) $(QT_CFLAGS) $(QSCI_CFLAGS) \ -I$(QT_INC)/Qsci \ - vendor/qt_shim.cpp \ + support/vendor-overrides/qt_shim.cpp \ -o libqt_shim.$(SHLIB_EXT) \ $(QT_LIBS) $(QSCI_LIBS) -lvterm $(PTY_LINK) -jerboa_qt_shim.$(SHLIB_EXT): vendor/qt_chez_shim.c vendor/qt_shim.h - gcc $(SHLIB_FLAGS) -O2 -o jerboa_qt_shim.$(SHLIB_EXT) vendor/qt_chez_shim.c -Ivendor -DQT_SCINTILLA_AVAILABLE -Wall +jerboa_qt_shim.$(SHLIB_EXT): support/vendor-overrides/qt_chez_shim.c support/vendor-overrides/qt_shim.h + gcc $(SHLIB_FLAGS) -O2 -o jerboa_qt_shim.$(SHLIB_EXT) support/vendor-overrides/qt_chez_shim.c -Isupport/vendor-overrides -DQT_SCINTILLA_AVAILABLE -Wall libjsh-ffi.$(SHLIB_EXT): $(JSH_ROOT)/ffi-shim.c gcc $(SHLIB_FLAGS) -O2 -o libjsh-ffi.$(SHLIB_EXT) $(JSH_ROOT)/ffi-shim.c -Wall @@ -813,7 +813,7 @@ ifeq ($(JEMACS_ENABLE_AWS),1) endif # Use stub if the Rust musl build hasn't been compiled yet (regular file check) _RUST_COREUTILS := $(JSH_SRC)/rust-coreutils/target/x86_64-unknown-linux-musl/release/libjsh_coreutils.a -JSH_COREUTILS_LIB ?= $(shell test -f $(_RUST_COREUTILS) && echo $(_RUST_COREUTILS) || echo $(CURDIR)/vendor/libjsh_coreutils_stub.a) +JSH_COREUTILS_LIB ?= $(shell test -f $(_RUST_COREUTILS) && echo $(_RUST_COREUTILS) || echo $(CURDIR)/support/vendor-overrides/libjsh_coreutils_stub.a) DEPS_IMAGE := jemacs-deps:$(ARCH) @@ -864,7 +864,7 @@ jemacs-linux-amd64-local: ensure-jerboa-tools vendor-deps @file jemacs-linux-amd64 jemacs-linux-amd64-remote: - $(SSH) $(LINUX_BUILD_HOST) 'cd $(LINUX_BUILD_ROOT) && git pull --ff-only && make linux-amd64' + $(SSH) $(LINUX_BUILD_HOST) 'set -e; if [ -d "$(LINUX_BUILD_ROOT)/.git" ]; then cd "$(LINUX_BUILD_ROOT)" && git fetch origin && git checkout "$(REMOTE_BRANCH)" && git pull --ff-only; else mkdir -p "$$(dirname "$(LINUX_BUILD_ROOT)")" && git clone "$(REMOTE_REPO_URL)" "$(LINUX_BUILD_ROOT)" && cd "$(LINUX_BUILD_ROOT)" && git checkout "$(REMOTE_BRANCH)"; fi; make linux-amd64' $(SCP) $(LINUX_BUILD_HOST):$(LINUX_BUILD_ROOT)/jemacs-linux-amd64 ./jemacs-linux-amd64 @ls -lh jemacs-linux-amd64 @file jemacs-linux-amd64 @@ -886,7 +886,7 @@ jemacs-linux-arm64-local: ensure-jerboa-tools vendor-deps @file jemacs-linux-arm64 jemacs-linux-arm64-remote: - $(SSH) $(LINUX_ARM64_BUILD_HOST) 'cd $(LINUX_ARM64_BUILD_ROOT) && git pull --ff-only && make linux-arm64' + $(SSH) $(LINUX_ARM64_BUILD_HOST) 'set -e; if [ -d "$(LINUX_ARM64_BUILD_ROOT)/.git" ]; then cd "$(LINUX_ARM64_BUILD_ROOT)" && git fetch origin && git checkout "$(REMOTE_BRANCH)" && git pull --ff-only; else mkdir -p "$$(dirname "$(LINUX_ARM64_BUILD_ROOT)")" && git clone "$(REMOTE_REPO_URL)" "$(LINUX_ARM64_BUILD_ROOT)" && cd "$(LINUX_ARM64_BUILD_ROOT)" && git checkout "$(REMOTE_BRANCH)"; fi; make linux-arm64' $(SCP) $(LINUX_ARM64_BUILD_HOST):$(LINUX_ARM64_BUILD_ROOT)/jemacs-linux-arm64 ./jemacs-linux-arm64 @ls -lh jemacs-linux-arm64 @file jemacs-linux-arm64 @@ -939,13 +939,13 @@ build-jemacs-qt-static: check-root rm -f /src/src/.jerbuild-hashes 2>/dev/null; \ cd /src && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ --script /deps/jerboa/jerbuild.ss src/ lib/ && \ - if [ -f /src/vendor/qt_shim.cpp ]; then \ + if [ -f /src/support/vendor-overrides/qt_shim.cpp ]; then \ echo "Rebuilding libqt_shim.a from updated qt_shim.cpp..." && \ QT_CFLAGS=$$(pkg-config --cflags Qt6Widgets 2>/dev/null || \ echo "-I/opt/qt6-static/include -I/opt/qt6-static/include/QtCore \ -I/opt/qt6-static/include/QtGui -I/opt/qt6-static/include/QtWidgets") && \ QSCI_FLAGS="-DQT_SCINTILLA_AVAILABLE -I/opt/qt6-static/include -I/opt/qt6-static/include/Qsci" && \ - cp /src/vendor/qt_shim.cpp /deps/qt-shim/qt_shim.cpp && \ + cp /src/support/vendor-overrides/qt_shim.cpp /deps/qt-shim/qt_shim.cpp && \ g++ -c -fPIC -std=c++17 -DJEMACS_CHEZ_SMP $$QT_CFLAGS $$QSCI_FLAGS \ /deps/qt-shim/qt_shim.cpp \ -o /deps/qt-shim/qt_shim_static.o && \ @@ -979,51 +979,51 @@ build-jemacs-qt-static: check-root printf '%s\n' '(import (jerboa-scintilla constants) (jerboa-scintilla scintilla) (jerboa-scintilla ffi) (jerboa-scintilla style) (jerboa-scintilla tui))' > /tmp/compile-jerboa-scintilla.ss && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa-scintilla/src:/deps/jerboa/lib \ --compile-imported-libraries --script /tmp/compile-jerboa-scintilla.ss && \ - cp /src/vendor/jerboa-net-tcp-static.sls /deps/jerboa/lib/std/net/tcp.sls && \ - cp /src/vendor/jerboa-net-tcp-raw-static.sls /deps/jerboa/lib/std/net/tcp-raw.sls && \ - cp /src/vendor/jerboa-net-uri.sls /deps/jerboa/lib/std/net/uri.sls && \ + cp /src/support/vendor-overrides/jerboa-net-tcp-static.sls /deps/jerboa/lib/std/net/tcp.sls && \ + cp /src/support/vendor-overrides/jerboa-net-tcp-raw-static.sls /deps/jerboa/lib/std/net/tcp-raw.sls && \ + cp /src/support/vendor-overrides/jerboa-net-uri.sls /deps/jerboa/lib/std/net/uri.sls && \ rm -f /deps/jerboa/lib/std/net/*.wpo /deps/jerboa/lib/std/net/*.so && \ - cp /src/vendor/jerboa-crypto-native-static.sls /deps/jerboa/lib/std/crypto/native.sls && \ + cp /src/support/vendor-overrides/jerboa-crypto-native-static.sls /deps/jerboa/lib/std/crypto/native.sls && \ rm -f /deps/jerboa/lib/std/crypto/*.wpo /deps/jerboa/lib/std/crypto/*.so && \ mkdir -p /deps/jerboa/lib/std/security && \ - cp /src/vendor/jerboa-security-capsicum-static.sls /deps/jerboa/lib/std/security/capsicum.sls && \ + cp /src/support/vendor-overrides/jerboa-security-capsicum-static.sls /deps/jerboa/lib/std/security/capsicum.sls && \ rm -f /deps/jerboa/lib/std/security/*.wpo /deps/jerboa/lib/std/security/*.so && \ - cp /src/vendor/jerboa-os-landlock-static.sls /deps/jerboa/lib/std/os/landlock.sls && \ + cp /src/support/vendor-overrides/jerboa-os-landlock-static.sls /deps/jerboa/lib/std/os/landlock.sls && \ rm -f /deps/jerboa/lib/std/os/landlock.wpo /deps/jerboa/lib/std/os/landlock.so && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-tcp.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-tcp.ss && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-tcp-raw.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-tcp-raw.ss && \ /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-uri.ss && \ - cp /src/vendor/jerboa-repl-static.sls /deps/jerboa/lib/std/repl.sls && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-uri.ss && \ + cp /src/support/vendor-overrides/jerboa-repl-static.sls /deps/jerboa/lib/std/repl.sls && \ rm -f /deps/jerboa/lib/std/repl.wpo /deps/jerboa/lib/std/repl.so && \ cd /deps/jerboa/lib && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-repl.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-repl.ss && \ cd /src && \ rm -f /src/lib/jerboa/*.wpo /src/lib/jerboa/*.so && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib:/deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-repl-socket.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-repl-socket.ss && \ echo "Building tree-sitter shim + queries..." && \ gcc -c -O2 -I/opt/tree-sitter-include -o /tmp/jemacs-build/treesitter_shim.o \ /src/support/treesitter_shim.c -Wall && \ gcc -c -O2 -o /tmp/jemacs-build/treesitter_queries.o \ /src/support/treesitter_queries.c -Wall && \ if [ "$(JEMACS_ENABLE_AWS)" = "1" ]; then \ - cp /src/vendor/jerboa-ssl-static.sls /deps/jerboa-ssl/src/jerboa-ssl.sls && \ - cp /src/vendor/jerboa-aws-crypto-native.sls /deps/jerboa-aws/jerboa-aws/crypto.sls && \ + cp /src/support/vendor-overrides/jerboa-ssl-static.sls /deps/jerboa-ssl/src/jerboa-ssl.sls && \ + cp /src/support/vendor-overrides/jerboa-aws-crypto-native.sls /deps/jerboa-aws/jerboa-aws/crypto.sls && \ find /deps/jerboa-ssl -name '*.so' -delete && find /deps/jerboa-ssl -name '*.wpo' -delete && \ find /deps/jerboa-https -name '*.so' -delete && find /deps/jerboa-https -name '*.wpo' -delete && \ find /deps/jerboa-aws -name '*.so' -delete && find /deps/jerboa-aws -name '*.wpo' -delete && \ JEMACS_STATIC=1 /opt/chez/bin/scheme \ --libdirs /deps/jerboa-ssl/src:/deps/jerboa/lib \ - --compile-imported-libraries -q --script /src/vendor/jerboa-ssl-compile-libs.ss && \ + --compile-imported-libraries -q --script /src/support/vendor-overrides/jerboa-ssl-compile-libs.ss && \ JEMACS_STATIC=1 /opt/chez/bin/scheme \ --libdirs /deps/jerboa-https/src:/deps/jerboa-ssl/src \ - --compile-imported-libraries -q --script /src/vendor/jerboa-https-compile-libs.ss && \ + --compile-imported-libraries -q --script /src/support/vendor-overrides/jerboa-https-compile-libs.ss && \ JEMACS_STATIC=1 /opt/chez/bin/scheme \ --libdirs /deps/jerboa-aws:/deps/jerboa-https/src:/deps/jerboa-ssl/src:/deps/jerboa/lib \ - --compile-imported-libraries -q --script /src/vendor/jerboa-aws-compile-libs.ss; \ + --compile-imported-libraries -q --script /src/support/vendor-overrides/jerboa-aws-compile-libs.ss; \ else \ echo "Skipping AWS static feature modules"; \ fi && \ @@ -1149,37 +1149,37 @@ build-jemacs-tui-static: check-root find /src/src/jerboa-emacs -name '*.ss' | sed 's|/src/src/|/src/lib/|; s|\.ss$$|.sls|' | xargs rm -f 2>/dev/null; \ cd /src && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ --script /deps/jerboa/jerbuild.ss src/ lib/ && \ - cp /src/vendor/jerboa-pcre2-ffi-static.ss /deps/jerboa-pcre2/jerboa-pcre2/ffi.ss && \ + cp /src/support/vendor-overrides/jerboa-pcre2-ffi-static.ss /deps/jerboa-pcre2/jerboa-pcre2/ffi.ss && \ /opt/chez/bin/scheme --libdirs /deps/jerboa-pcre2 \ - --compile-imported-libraries --script /src/vendor/jerboa-pcre2-compile-libs.ss && \ - cp /src/vendor/jerboa-scintilla-ffi-static.sls /deps/jerboa-scintilla/src/jerboa-scintilla/ffi.sls && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-pcre2-compile-libs.ss && \ + cp /src/support/vendor-overrides/jerboa-scintilla-ffi-static.sls /deps/jerboa-scintilla/src/jerboa-scintilla/ffi.sls && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa-scintilla/src \ - --compile-imported-libraries --script /src/vendor/jerboa-scintilla-compile-libs.ss && \ - cp /src/vendor/jerboa-net-tcp-static.sls /deps/jerboa/lib/std/net/tcp.sls && \ - cp /src/vendor/jerboa-net-tcp-raw-static.sls /deps/jerboa/lib/std/net/tcp-raw.sls && \ - cp /src/vendor/jerboa-net-uri.sls /deps/jerboa/lib/std/net/uri.sls && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-scintilla-compile-libs.ss && \ + cp /src/support/vendor-overrides/jerboa-net-tcp-static.sls /deps/jerboa/lib/std/net/tcp.sls && \ + cp /src/support/vendor-overrides/jerboa-net-tcp-raw-static.sls /deps/jerboa/lib/std/net/tcp-raw.sls && \ + cp /src/support/vendor-overrides/jerboa-net-uri.sls /deps/jerboa/lib/std/net/uri.sls && \ rm -f /deps/jerboa/lib/std/net/*.wpo /deps/jerboa/lib/std/net/*.so && \ - cp /src/vendor/jerboa-crypto-native-static.sls /deps/jerboa/lib/std/crypto/native.sls && \ + cp /src/support/vendor-overrides/jerboa-crypto-native-static.sls /deps/jerboa/lib/std/crypto/native.sls && \ rm -f /deps/jerboa/lib/std/crypto/*.wpo /deps/jerboa/lib/std/crypto/*.so && \ mkdir -p /deps/jerboa/lib/std/security && \ - cp /src/vendor/jerboa-security-capsicum-static.sls /deps/jerboa/lib/std/security/capsicum.sls && \ + cp /src/support/vendor-overrides/jerboa-security-capsicum-static.sls /deps/jerboa/lib/std/security/capsicum.sls && \ rm -f /deps/jerboa/lib/std/security/*.wpo /deps/jerboa/lib/std/security/*.so && \ - cp /src/vendor/jerboa-os-landlock-static.sls /deps/jerboa/lib/std/os/landlock.sls && \ + cp /src/support/vendor-overrides/jerboa-os-landlock-static.sls /deps/jerboa/lib/std/os/landlock.sls && \ rm -f /deps/jerboa/lib/std/os/landlock.wpo /deps/jerboa/lib/std/os/landlock.so && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-tcp.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-tcp.ss && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-tcp-raw.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-tcp-raw.ss && \ /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-uri.ss && \ - cp /src/vendor/jerboa-repl-static.sls /deps/jerboa/lib/std/repl.sls && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-uri.ss && \ + cp /src/support/vendor-overrides/jerboa-repl-static.sls /deps/jerboa/lib/std/repl.sls && \ rm -f /deps/jerboa/lib/std/repl.wpo /deps/jerboa/lib/std/repl.so && \ cd /deps/jerboa/lib && /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-repl.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-repl.ss && \ cd /src && \ rm -f /src/lib/jerboa/*.wpo /src/lib/jerboa/*.so && \ JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib:/deps/jerboa/lib \ - --compile-imported-libraries --script /src/vendor/jerboa-compile-repl-socket.ss && \ + --compile-imported-libraries --script /src/support/vendor-overrides/jerboa-compile-repl-socket.ss && \ echo "Building Scintilla+termbox+Lexilla static archives from source..." && \ make -C /deps/sci-vendor/scintilla/termbox/termbox_next && \ make -C /deps/sci-vendor/scintilla/termbox && \ @@ -1300,41 +1300,41 @@ linux-tui-local: vendor-deps | $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib -q rm -f $(MUSL_JERBOA)/lib/jerboa/*.wpo $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib --script $(MUSL_JERBOA)/jerbuild.ss src/ lib/ - cp vendor/jerboa-pcre2-ffi-static.ss $(MUSL_PCRE2)/jerboa-pcre2/ffi.ss + cp support/vendor-overrides/jerboa-pcre2-ffi-static.ss $(MUSL_PCRE2)/jerboa-pcre2/ffi.ss $(MUSL_SCHEME) --libdirs $(MUSL_PCRE2) \ - --compile-imported-libraries --script vendor/jerboa-pcre2-compile-libs.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-pcre2-compile-libs.ss rm -f $(MUSL_PCRE2)/jerboa-pcre2/*.wpo - cp vendor/jerboa-scintilla-ffi-static.sls $(MUSL_SCINTILLA)/src/jerboa-scintilla/ffi.sls + cp support/vendor-overrides/jerboa-scintilla-ffi-static.sls $(MUSL_SCINTILLA)/src/jerboa-scintilla/ffi.sls JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_SCINTILLA)/src \ - --compile-imported-libraries --script vendor/jerboa-scintilla-compile-libs.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-scintilla-compile-libs.ss rm -f $(MUSL_SCINTILLA)/src/jerboa-scintilla/*.wpo - cp vendor/jerboa-net-tcp-static.sls $(MUSL_JERBOA)/lib/std/net/tcp.sls - cp vendor/jerboa-net-tcp-raw-static.sls $(MUSL_JERBOA)/lib/std/net/tcp-raw.sls - cp vendor/jerboa-net-uri.sls $(MUSL_JERBOA)/lib/std/net/uri.sls - cp vendor/jerboa-net-tls-rustls-static.sls $(MUSL_JERBOA)/lib/std/net/tls-rustls.sls + cp support/vendor-overrides/jerboa-net-tcp-static.sls $(MUSL_JERBOA)/lib/std/net/tcp.sls + cp support/vendor-overrides/jerboa-net-tcp-raw-static.sls $(MUSL_JERBOA)/lib/std/net/tcp-raw.sls + cp support/vendor-overrides/jerboa-net-uri.sls $(MUSL_JERBOA)/lib/std/net/uri.sls + cp support/vendor-overrides/jerboa-net-tls-rustls-static.sls $(MUSL_JERBOA)/lib/std/net/tls-rustls.sls rm -f $(MUSL_JERBOA)/lib/std/net/*.wpo $(MUSL_JERBOA)/lib/std/net/*.so - cp vendor/jerboa-crypto-native-static.sls $(MUSL_JERBOA)/lib/std/crypto/native.sls + cp support/vendor-overrides/jerboa-crypto-native-static.sls $(MUSL_JERBOA)/lib/std/crypto/native.sls rm -f $(MUSL_JERBOA)/lib/std/crypto/*.wpo $(MUSL_JERBOA)/lib/std/crypto/*.so mkdir -p $(MUSL_JERBOA)/lib/std/security - cp vendor/jerboa-security-capsicum-static.sls $(MUSL_JERBOA)/lib/std/security/capsicum.sls + cp support/vendor-overrides/jerboa-security-capsicum-static.sls $(MUSL_JERBOA)/lib/std/security/capsicum.sls rm -f $(MUSL_JERBOA)/lib/std/security/*.wpo $(MUSL_JERBOA)/lib/std/security/*.so - cp vendor/jerboa-os-landlock-static.sls $(MUSL_JERBOA)/lib/std/os/landlock.sls + cp support/vendor-overrides/jerboa-os-landlock-static.sls $(MUSL_JERBOA)/lib/std/os/landlock.sls rm -f $(MUSL_JERBOA)/lib/std/os/landlock.wpo $(MUSL_JERBOA)/lib/std/os/landlock.so JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-tcp.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-tcp.ss JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-tcp-raw.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-tcp-raw.ss $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-uri.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-uri.ss rm -f $(MUSL_JERBOA)/lib/std/net/*.wpo - cp vendor/jerboa-repl-static.sls $(MUSL_JERBOA)/lib/std/repl.sls + cp support/vendor-overrides/jerboa-repl-static.sls $(MUSL_JERBOA)/lib/std/repl.sls rm -f $(MUSL_JERBOA)/lib/std/repl.wpo $(MUSL_JERBOA)/lib/std/repl.so cd $(MUSL_JERBOA)/lib && $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script $(CURDIR)/vendor/jerboa-compile-repl.ss + --compile-imported-libraries --script $(CURDIR)/support/vendor-overrides/jerboa-compile-repl.ss rm -f $(MUSL_JERBOA)/lib/std/repl.wpo rm -f lib/jerboa/*.wpo lib/jerboa/*.so JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs lib \ - --compile-imported-libraries --script vendor/jerboa-compile-repl-socket.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-repl-socket.ss rm -f lib/jerboa/*.wpo find $(MUSL_JSH_ROOT) -name '*.wpo' -delete 2>/dev/null; true JEMACS_STATIC=1 \ @@ -1406,60 +1406,60 @@ linux-qt-local: vendor-deps JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs /tmp/jemacs-build/jerboa-qt \ --compile-imported-libraries --script /tmp/jemacs-build/jerboa-qt/compile-libs.ss rm -f /tmp/jemacs-build/jerboa-qt/jerboa-qt/*.wpo - cp vendor/jerboa-pcre2-ffi-static.ss $(MUSL_PCRE2)/jerboa-pcre2/ffi.ss + cp support/vendor-overrides/jerboa-pcre2-ffi-static.ss $(MUSL_PCRE2)/jerboa-pcre2/ffi.ss $(MUSL_SCHEME) --libdirs $(MUSL_PCRE2) \ - --compile-imported-libraries --script vendor/jerboa-pcre2-compile-libs.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-pcre2-compile-libs.ss rm -f $(MUSL_PCRE2)/jerboa-pcre2/*.wpo - cp vendor/jerboa-scintilla-ffi-static.sls $(MUSL_SCINTILLA)/src/jerboa-scintilla/ffi.sls + cp support/vendor-overrides/jerboa-scintilla-ffi-static.sls $(MUSL_SCINTILLA)/src/jerboa-scintilla/ffi.sls JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_SCINTILLA)/src \ - --compile-imported-libraries --script vendor/jerboa-scintilla-compile-libs.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-scintilla-compile-libs.ss rm -f $(MUSL_SCINTILLA)/src/jerboa-scintilla/*.wpo - cp vendor/jerboa-net-tcp-static.sls $(MUSL_JERBOA)/lib/std/net/tcp.sls - cp vendor/jerboa-net-tcp-raw-static.sls $(MUSL_JERBOA)/lib/std/net/tcp-raw.sls - cp vendor/jerboa-net-uri.sls $(MUSL_JERBOA)/lib/std/net/uri.sls - cp vendor/jerboa-net-tls-rustls-static.sls $(MUSL_JERBOA)/lib/std/net/tls-rustls.sls + cp support/vendor-overrides/jerboa-net-tcp-static.sls $(MUSL_JERBOA)/lib/std/net/tcp.sls + cp support/vendor-overrides/jerboa-net-tcp-raw-static.sls $(MUSL_JERBOA)/lib/std/net/tcp-raw.sls + cp support/vendor-overrides/jerboa-net-uri.sls $(MUSL_JERBOA)/lib/std/net/uri.sls + cp support/vendor-overrides/jerboa-net-tls-rustls-static.sls $(MUSL_JERBOA)/lib/std/net/tls-rustls.sls rm -f $(MUSL_JERBOA)/lib/std/net/*.wpo $(MUSL_JERBOA)/lib/std/net/*.so - cp vendor/jerboa-crypto-native-static.sls $(MUSL_JERBOA)/lib/std/crypto/native.sls + cp support/vendor-overrides/jerboa-crypto-native-static.sls $(MUSL_JERBOA)/lib/std/crypto/native.sls rm -f $(MUSL_JERBOA)/lib/std/crypto/*.wpo $(MUSL_JERBOA)/lib/std/crypto/*.so mkdir -p $(MUSL_JERBOA)/lib/std/security - cp vendor/jerboa-security-capsicum-static.sls $(MUSL_JERBOA)/lib/std/security/capsicum.sls + cp support/vendor-overrides/jerboa-security-capsicum-static.sls $(MUSL_JERBOA)/lib/std/security/capsicum.sls rm -f $(MUSL_JERBOA)/lib/std/security/*.wpo $(MUSL_JERBOA)/lib/std/security/*.so - cp vendor/jerboa-os-landlock-static.sls $(MUSL_JERBOA)/lib/std/os/landlock.sls + cp support/vendor-overrides/jerboa-os-landlock-static.sls $(MUSL_JERBOA)/lib/std/os/landlock.sls rm -f $(MUSL_JERBOA)/lib/std/os/landlock.wpo $(MUSL_JERBOA)/lib/std/os/landlock.so JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-tcp.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-tcp.ss JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-tcp-raw.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-tcp-raw.ss $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script vendor/jerboa-compile-uri.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-uri.ss rm -f $(MUSL_JERBOA)/lib/std/net/*.wpo - cp vendor/jerboa-repl-static.sls $(MUSL_JERBOA)/lib/std/repl.sls + cp support/vendor-overrides/jerboa-repl-static.sls $(MUSL_JERBOA)/lib/std/repl.sls rm -f $(MUSL_JERBOA)/lib/std/repl.wpo $(MUSL_JERBOA)/lib/std/repl.so cd $(MUSL_JERBOA)/lib && $(MUSL_SCHEME) --libdirs $(MUSL_JERBOA)/lib \ - --compile-imported-libraries --script $(CURDIR)/vendor/jerboa-compile-repl.ss + --compile-imported-libraries --script $(CURDIR)/support/vendor-overrides/jerboa-compile-repl.ss rm -f $(MUSL_JERBOA)/lib/std/repl.wpo rm -f lib/jerboa/*.wpo lib/jerboa/*.so JEMACS_STATIC=1 $(MUSL_SCHEME) --libdirs lib \ - --compile-imported-libraries --script vendor/jerboa-compile-repl-socket.ss + --compile-imported-libraries --script support/vendor-overrides/jerboa-compile-repl-socket.ss rm -f lib/jerboa/*.wpo # Compile optional SSL/HTTPS/AWS feature modules @if [ "$(JEMACS_ENABLE_AWS)" = "1" ]; then \ - cp vendor/jerboa-ssl-static.sls vendor/jerboa-ssl/src/jerboa-ssl.sls && \ - cp vendor/jerboa-aws-crypto-native.sls $(JAWS)/jerboa-aws/crypto.sls && \ + cp support/vendor-overrides/jerboa-ssl-static.sls vendor/jerboa-ssl/src/jerboa-ssl.sls && \ + cp support/vendor-overrides/jerboa-aws-crypto-native.sls $(JAWS)/jerboa-aws/crypto.sls && \ find vendor/jerboa-ssl -name '*.so' -delete; find vendor/jerboa-ssl -name '*.wpo' -delete; \ find vendor/jerboa-https -name '*.so' -delete; find vendor/jerboa-https -name '*.wpo' -delete; \ find vendor/jerboa-aws -name '*.so' -delete; find vendor/jerboa-aws -name '*.wpo' -delete; \ JEMACS_STATIC=1 $(MUSL_SCHEME) \ --libdirs vendor/jerboa-ssl/src:$(MUSL_JERBOA)/lib \ - --compile-imported-libraries -q --script vendor/jerboa-ssl-compile-libs.ss && \ + --compile-imported-libraries -q --script support/vendor-overrides/jerboa-ssl-compile-libs.ss && \ find vendor/jerboa-ssl -name '*.wpo' -delete && \ JEMACS_STATIC=1 $(MUSL_SCHEME) \ --libdirs vendor/jerboa-https/src:vendor/jerboa-ssl/src \ - --compile-imported-libraries -q --script vendor/jerboa-https-compile-libs.ss && \ + --compile-imported-libraries -q --script support/vendor-overrides/jerboa-https-compile-libs.ss && \ find vendor/jerboa-https -name '*.wpo' -delete && \ JEMACS_STATIC=1 $(MUSL_SCHEME) \ --libdirs $(JAWS):$(CHTTPS):$(CSSL):$(MUSL_JERBOA)/lib \ - --compile-imported-libraries -q --script vendor/jerboa-aws-compile-libs.ss && \ + --compile-imported-libraries -q --script support/vendor-overrides/jerboa-aws-compile-libs.ss && \ find vendor/jerboa-aws -name '*.wpo' -delete; \ else \ echo "Skipping AWS static feature modules"; \ @@ -1480,7 +1480,7 @@ linux-qt-local: vendor-deps JERBOA_SCINTILLA_DIR=$(MUSL_SCINTILLA)/src \ JERBOA_QT_DIR=/tmp/jemacs-build/jerboa-qt \ JERBOA_QT_SHIM_DIR=/opt/qt-shim \ - JSH_COREUTILS_LIB=vendor/libjsh_coreutils_stub.a \ + JSH_COREUTILS_LIB=support/vendor-overrides/libjsh_coreutils_stub.a \ $(AWS_LOCAL_STATIC_ENV) \ TREE_SITTER_INCLUDE=/opt/tree-sitter-include \ TREE_SITTER_LIB=/opt/tree-sitter-lib \ --- a/build-binary-qt.ss +++ b/build-binary-qt.ss @@ -117,7 +117,7 @@ (repo-path "vendor/jerboa-qt"))) (define qt-shim-dir (or (getenv "JERBOA_QT_SHIM_DIR") - (repo-path "vendor"))) + (repo-path "support/vendor-overrides"))) (define jaws-dir (or (getenv "JAWS_DIR") (repo-path "vendor/jerboa-aws/lib"))) @@ -336,7 +336,7 @@ ;; jerboa_qt_shim — use jerboa-emacs deferred-callback version (avoids Sactivate_thread GC deadlock) (let* ((qt-cflags (shell-output "pkg-config --cflags Qt6Widgets" ""))) ;; Static .o for static builds - (let ((cmd (format "gcc -c -O2 -DQT_SCINTILLA_AVAILABLE -o jemacs-qt-chez-shim.o vendor/qt_chez_shim.c -I~a ~a -Wall 2>&1" + (let ((cmd (format "gcc -c -O2 -DQT_SCINTILLA_AVAILABLE -o jemacs-qt-chez-shim.o support/vendor-overrides/qt_chez_shim.c -I~a ~a -Wall 2>&1" qt-shim-dir qt-cflags))) (unless (= 0 (system cmd)) (display "Error: jerboa_qt_shim compilation failed\n") @@ -346,7 +346,7 @@ (let* ((shlib-flags (if macos-build? "-dynamiclib -Wl,-undefined,dynamic_lookup" "-shared -fPIC")) - (cmd (format "gcc ~a -O2 -DQT_SCINTILLA_AVAILABLE -o jerboa_qt_shim.~a vendor/qt_chez_shim.c -I~a ~a -Wall 2>&1" + (cmd (format "gcc ~a -O2 -DQT_SCINTILLA_AVAILABLE -o jerboa_qt_shim.~a support/vendor-overrides/qt_chez_shim.c -I~a ~a -Wall 2>&1" shlib-flags shlib-ext qt-shim-dir qt-cflags))) (unless (= 0 (system cmd)) (display "Error: jerboa_qt_shim shlib compilation failed\n") --- a/docs/stress-test.md +++ b/docs/stress-test.md @@ -28,7 +28,7 @@ information — it just dies. We need two things: ### Design -Added to `vendor/qt_shim.cpp`, the crash reporter has three components: +Added to `support/vendor-overrides/qt_shim.cpp`, the crash reporter has three components: #### 1. FFI Call Ring Buffer --- a/plan-android.md +++ b/plan-android.md @@ -133,6 +133,6 @@ The fundamental architecture of jerboa-emacs (embedded Chez + static Qt + C FFI - `Makefile` — New `android-qt` target - `Dockerfile` — New Android NDK variant (or separate `Dockerfile.android`) - `build-binary-qt.ss` — Target machine type (`tarm64le` instead of `ta6le`) -- `vendor/qt_shim.cpp` — Minor `#ifdef __ANDROID__` guards if needed +- `support/vendor-overrides/qt_shim.cpp` — Minor `#ifdef __ANDROID__` guards if needed - `src/jerboa-emacs/qt/app.ss` — Android lifecycle hooks - `src/jerboa-emacs/qt/sci-shim.ss` — Touch/DPI adaptations new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-aws-compile-libs.ss @@ -0,0 +1,18 @@ +#!chezscheme +;; Compile jerboa-aws EC2 modules needed by jemacs +(import (chezscheme)) +(compile-imported-libraries #t) +(generate-wpo-files #t) +(import (jerboa-aws creds)) +(import (jerboa-aws crypto)) +(import (jerboa-aws xml)) +(import (jerboa-aws json)) +(import (jerboa-aws time)) +(import (jerboa-aws uri)) +(import (jerboa-aws sigv4)) +(import (jerboa-aws request)) +(import (jerboa-aws api)) +(import (jerboa-aws ec2 xml)) +(import (jerboa-aws ec2 params)) +(import (jerboa-aws ec2 api)) +(import (jerboa-aws ec2 instances)) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-aws-crypto-native.sls @@ -0,0 +1,33 @@ +#!chezscheme +;;; (jerboa-aws crypto) — SHA256/HMAC via std/crypto/native (no subprocess) +;;; Replaces the default jerboa-aws/crypto which shells out to openssl CLI. +;;; Used in jemacs static builds where there is no external openssl binary. + +(library (jerboa-aws crypto) + (export sha256 hex-encode hmac-sha256) + (import (chezscheme) + (std crypto native)) + + ;; SHA256: returns raw 32-byte bytevector (same contract as original) + (define (sha256 data) + (native-digest 'sha256 (if (string? data) (string->utf8 data) data))) + + ;; Hex-encode a bytevector to a lowercase hex string + (define (hex-encode bv) + (let* ([len (bytevector-length bv)] + [hex (make-string (* len 2))]) + (do ([i 0 (+ i 1)]) + ((= i len) hex) + (let* ([b (bytevector-u8-ref bv i)] + [hi (fxsra b 4)] + [lo (fxand b #xf)]) + (string-set! hex (* i 2) (string-ref "0123456789abcdef" hi)) + (string-set! hex (+ (* i 2) 1) (string-ref "0123456789abcdef" lo)))))) + + ;; HMAC-SHA256: returns raw 32-byte bytevector + (define (hmac-sha256 key data) + (let ([key-bv (if (string? key) (string->utf8 key) key)] + [data-bv (if (string? data) (string->utf8 data) data)]) + (native-hmac-sha256 key-bv data-bv))) + + ) ;; end library new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-compile-repl-socket.ss @@ -0,0 +1,5 @@ +;;; jerboa-compile-repl-socket.ss — Compile jerboa/repl-socket for static build +(import (chezscheme)) +(compile-imported-libraries #t) +(generate-wpo-files #t) +(import (jerboa repl-socket)) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-compile-repl.ss @@ -0,0 +1,5 @@ +#!chezscheme +(import (chezscheme)) +(parameterize ((compile-imported-libraries #t) + (generate-wpo-files #t)) + (compile-library "std/repl.sls")) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-compile-tcp-raw.ss @@ -0,0 +1,5 @@ +;;; jerboa-compile-tcp-raw.ss — Recompile std/net/tcp-raw after patching for static build +(import (chezscheme)) +(compile-imported-libraries #t) +(generate-wpo-files #t) +(import (std net tcp-raw)) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-compile-tcp.ss @@ -0,0 +1,5 @@ +;;; jerboa-compile-tcp.ss — Recompile std/net/tcp after patching for static build +(import (chezscheme)) +(compile-imported-libraries #t) +(generate-wpo-files #t) +(import (std net tcp)) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-compile-uri.ss @@ -0,0 +1,5 @@ +;;; jerboa-compile-uri.ss — Compile std/net/uri for static build +(import (chezscheme)) +(compile-imported-libraries #t) +(generate-wpo-files #t) +(import (std net uri)) new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-crypto-native-static.sls @@ -0,0 +1,31 @@ +#!chezscheme +;;; (std crypto native) — Static build stub +;;; Crypto not available in static builds (no OpenSSL linked). +;;; All functions error with "libcrypto not available". + +(library (std crypto native) + (export + native-md5 native-sha1 native-sha256 native-sha384 native-sha512 + native-digest + native-random-bytes + native-random-bytes! + native-hmac-sha256 + native-crypto-memcmp) + + (import (chezscheme)) + + (define (not-available who) + (error who "libcrypto not available in static build")) + + (define (native-md5 data) (not-available 'native-md5)) + (define (native-sha1 data) (not-available 'native-sha1)) + (define (native-sha256 data) (not-available 'native-sha256)) + (define (native-sha384 data) (not-available 'native-sha384)) + (define (native-sha512 data) (not-available 'native-sha512)) + (define (native-digest algo data) (not-available 'native-digest)) + (define (native-random-bytes n) (not-available 'native-random-bytes)) + (define (native-random-bytes! bv) (not-available 'native-random-bytes!)) + (define (native-hmac-sha256 key data) (not-available 'native-hmac-sha256)) + (define (native-crypto-memcmp a b) (not-available 'native-crypto-memcmp)) + + ) ;; end library new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-net-tcp-raw-static.sls @@ -0,0 +1,172 @@ +#!chezscheme +;;; (std net tcp-raw) — Raw fd-based TCP, jerboa-ssl compatible API +;;; +;;; Static-build version: skips load-shared-object (symbols in static binary). + +(library (std net tcp-raw) + (export + tcp-connect tcp-listen tcp-accept tcp-close + tcp-read tcp-write tcp-write-string tcp-read-all + tcp-set-timeout) + + (import (chezscheme)) + + ;; ========== FFI ========== + + ;; In static builds, POSIX symbols are already linked in. + (define _libc + (let ((v (getenv "JEMACS_STATIC"))) + (if (and v (not (string=? v "")) (not (string=? v "0"))) + #f ; symbols already in static binary + (guard (exn [#t (void)]) + (load-shared-object "libc.so.6"))))) + + (define c-socket (foreign-procedure "socket" (int int int) int)) + (define c-bind (foreign-procedure "bind" (int void* int) int)) + (define c-listen (foreign-procedure "listen" (int int) int)) + (define c-accept (foreign-procedure "accept" (int void* void*) int)) + (define c-connect (foreign-procedure "connect" (int void* int) int)) + (define c-close (foreign-procedure "close" (int) int)) + (define c-setsockopt (foreign-procedure "setsockopt" (int int int void* int) int)) + (define c-read (foreign-procedure "read" (int u8* size_t) ssize_t)) + (define c-write (foreign-procedure "write" (int u8* size_t) ssize_t)) + (define c-htons (foreign-procedure "htons" (unsigned-short) unsigned-short)) + (define c-inet-pton (foreign-procedure "inet_pton" (int string void*) int)) + + ;; errno + (define c-errno-location (foreign-procedure "__errno_location" () void*)) + (define (get-errno) (foreign-ref 'int (c-errno-location) 0)) + (define EINTR 4) + + ;; Constants + (define AF_INET 2) + (define SOCK_STREAM 1) + (define SOL_SOCKET 1) + (define SO_REUSEADDR 2) + (define SO_RCVTIMEO 20) + (define SO_SNDTIMEO 21) + (define SOCKADDR_IN_SIZE 16) + + ;; ========== sockaddr_in ========== + + (define (make-sockaddr-in address port) + (let ([buf (foreign-alloc SOCKADDR_IN_SIZE)]) + (let lp ([i 0]) + (when (< i SOCKADDR_IN_SIZE) + (foreign-set! 'unsigned-8 buf i 0) + (lp (+ i 1)))) + (foreign-set! 'unsigned-short buf 0 AF_INET) + (foreign-set! 'unsigned-short buf 2 (c-htons port)) + (let ([addr-ptr (+ buf 4)]) + (when (= (c-inet-pton AF_INET address addr-ptr) 0) + (foreign-free buf) + (error 'make-sockaddr-in "invalid address" address))) + buf)) + + ;; ========== API (jerboa-ssl compatible) ========== + + (define (tcp-connect host port) + (let ([fd (c-socket AF_INET SOCK_STREAM 0)]) + (when (< fd 0) + (error 'tcp-connect "socket() failed")) + (let ([addr (make-sockaddr-in host port)]) + (let loop () + (let ([rc (c-connect fd addr SOCKADDR_IN_SIZE)]) + (cond + [(>= rc 0) + (foreign-free addr) + fd] + [(= (get-errno) EINTR) (loop)] + [else + (foreign-free addr) + (c-close fd) + (error 'tcp-connect "connect() failed" host port)])))))) + + (define tcp-listen + (case-lambda + [(port) (tcp-listen port 128)] + [(port backlog) + (let ([fd (c-socket AF_INET SOCK_STREAM 0)]) + (when (< fd 0) + (error 'tcp-listen "socket() failed")) + (let ([one (foreign-alloc 4)]) + (foreign-set! 'int one 0 1) + (c-setsockopt fd SOL_SOCKET SO_REUSEADDR one 4) + (foreign-free one)) + (let ([addr (make-sockaddr-in "0.0.0.0" port)]) + (let ([rc (c-bind fd addr SOCKADDR_IN_SIZE)]) + (foreign-free addr) + (when (< rc 0) + (c-close fd) + (error 'tcp-listen "bind() failed" port)))) + (when (< (c-listen fd backlog) 0) + (c-close fd) + (error 'tcp-listen "listen() failed")) + fd)])) + + (define (tcp-accept listen-fd) + (let loop () + (let ([client-fd (c-accept listen-fd 0 0)]) + (cond + [(>= client-fd 0) + (values client-fd "")] + [(= (get-errno) EINTR) (loop)] + [else (error 'tcp-accept "accept() failed")])))) + + (define (tcp-close fd) + (c-close fd)) + + (define (tcp-read fd buf len) + (let loop () + (let ([n (c-read fd buf len)]) + (cond + [(>= n 0) n] + [(= (get-errno) EINTR) (loop)] + [else -1])))) + + (define (tcp-write fd bv) + (let ([total (bytevector-length bv)]) + (let loop ([offset 0]) + (when (< offset total) + (let ([buf (if (= offset 0) bv + (let ([tmp (make-bytevector (- total offset))]) + (bytevector-copy! bv offset tmp 0 (- total offset)) + tmp))]) + (let ([n (c-write fd buf (- total offset))]) + (cond + [(> n 0) (loop (+ offset n))] + [(= (get-errno) EINTR) (loop offset)] + [else (error 'tcp-write "write failed")]))))))) + + (define (tcp-write-string fd str) + (tcp-write fd (string->utf8 str))) + + (define (tcp-read-all fd) + (let ([chunks '()] [total 0]) + (let loop () + (let ([buf (make-bytevector 4096)]) + (let ([n (tcp-read fd buf 4096)]) + (cond + [(> n 0) + (set! chunks (cons (cons buf n) chunks)) + (set! total (+ total n)) + (loop)] + [else + (let ([result (make-bytevector total)]) + (let lp ([cs (reverse chunks)] [off 0]) + (unless (null? cs) + (let ([bv (caar cs)] [n (cdar cs)]) + (bytevector-copy! bv 0 result off n) + (lp (cdr cs) (+ off n))))) + result)])))))) + + (define (tcp-set-timeout fd read-secs write-secs) + (let ([tv (foreign-alloc 16)]) + (foreign-set! 'long tv 0 read-secs) + (foreign-set! 'long tv 8 0) + (c-setsockopt fd SOL_SOCKET SO_RCVTIMEO tv 16) + (foreign-set! 'long tv 0 write-secs) + (c-setsockopt fd SOL_SOCKET SO_SNDTIMEO tv 16) + (foreign-free tv))) + + ) ;; end library new file mode 100644 --- /dev/null +++ b/support/vendor-overrides/jerboa-net-tcp-static.sls @@ -0,0 +1,232 @@ +#!chezscheme +;;; :std/net/tcp -- TCP client and server sockets +;;; +;;; Provides Gerbil-compatible TCP networking: +;;; (tcp-listen address port) → server socket +;;; (tcp-accept server) → (values input-port output-port) +;;; (tcp-connect address port) → (values input-port output-port) +;;; (tcp-close server) → void +;;; +;;; Ports are standard Chez Scheme binary ports transcoded to UTF-8. +;;; Use with-tcp-server for automatic cleanup. + +(library (std net tcp) + (export + tcp-listen tcp-accept tcp-close + tcp-connect + tcp-server? tcp-server-port + with-tcp-server) + + (import (chezscheme)) + + ;; ========== FFI ========== + + ;; Load libc for POSIX socket functions + (define _libc-loaded + (let ((v (getenv "JEMACS_STATIC"))) + (if (and v (not (string=? v "")) (not (string=? v "0"))) + #f ; symbols already in static binary + (load-shared-object #f)))) + + (define c-socket (foreign-procedure "socket" (int int int) int)) + (define c-bind (foreign-procedure "bind" (int void* int) int)) + (define c-listen (foreign-procedure "listen" (int int) int)) + (define c-accept (foreign-procedure "accept" (int void* void*) int)) + (define c-connect (foreign-procedure "connect" (int void* int) int)) + (define c-close (foreign-procedure "close" (int) int)) + (define c-setsockopt (foreign-procedure "setsockopt" (int int int void* int) int)) + (define c-read (foreign-procedure "read" (int u8* size_t) ssize_t)) + (define c-write (foreign-procedure "write" (int u8* size_t) ssize_t)) + (define c-htons (foreign-procedure "htons" (unsigned-short) unsigned-short)) + (define c-inet-pton (foreign-procedure "inet_pton" (int string void*) int)) + (define c-getsockname (foreign-procedure "getsockname" (int void* void*) int)) + + ;; errno access for EINTR retry + (define c-errno-location (foreign-procedure "__errno_location" () void*)) + (define (get-errno) (foreign-ref 'int (c-errno-location) 0)) + (define EINTR 4) + + ;; Constants + (define AF_INET 2) + (define SOCK_STREAM 1) + (define SOL_SOCKET 1) + (define SO_REUSEADDR 2) + (define SOCKADDR_IN_SIZE 16) ;; sizeof(struct sockaddr_in) on Linux + + ;; ========== sockaddr_in helpers ========== + + (define (make-sockaddr-in address port) + ;; Build a struct sockaddr_in in foreign memory + (let ([buf (foreign-alloc SOCKADDR_IN_SIZE)]) + ;; Zero the struct + (let lp ([i 0]) + (when (< i SOCKADDR_IN_SIZE) + (foreign-set! 'unsigned-8 buf i 0) + (lp (+ i 1)))) + ;; sin_family = AF_INET (offset 0, 2 bytes) + (foreign-set! 'unsigned-short buf 0 AF_INET) + ;; sin_port = htons(port) (offset 2, 2 bytes) + (foreign-set! 'unsigned-short buf 2 (c-htons port)) + ;; sin_addr (offset 4, 4 bytes) — parse address string + (let ([addr-ptr (+ buf 4)]) + (when (= (c-inet-pton AF_INET address addr-ptr) 0) + (foreign-free buf) + (error 'make-sockaddr-in "invalid address" address))) + buf)) + + (define (sockaddr-in-port buf) + ;; Extract port from sockaddr_in (in host byte order) + ;; sin_port is at offset 2, in network byte order + (let ([hi (foreign-ref 'unsigned-8 buf 2)] + [lo (foreign-ref 'unsigned-8 buf 3)]) + (+ (* hi 256) lo))) + + ;; ========== TCP Server ========== + + (define-record-type tcp-server + (fields + (immutable fd) + (mutable port-num)) ;; actual port (useful when 0 = OS-assigned) + (sealed #t)) + + (define tcp-listen + (case-lambda + [(address port) + (tcp-listen address port 128)] + [(address port backlog) + (let ([fd (c-socket AF_INET SOCK_STREAM 0)]) + (when (< fd 0) + (error 'tcp-listen "socket() failed")) + ;; SO_REUSEADDR + (let ([one (foreign-alloc 4)]) + (foreign-set! 'int one 0 1) + (c-setsockopt fd SOL_SOCKET SO_REUSEADDR one 4) + (foreign-free one)) + ;; Bind + (let ([addr (make-sockaddr-in address port)]) + (let ([rc (c-bind fd addr SOCKADDR_IN_SIZE)]) + (foreign-free addr) + (when (< rc 0) + (c-close fd) + (error 'tcp-listen "bind() failed" address port)))) + ;; Listen + (when (< (c-listen fd backlog) 0) + (c-close fd) + (error 'tcp-listen "listen() failed")) + ;; Get actual port (important when port=0) + (let ([actual-port + (let ([buf (foreign-alloc SOCKADDR_IN_SIZE)] + [len (foreign-alloc 4)]) + (foreign-set! 'int len 0 SOCKADDR_IN_SIZE) + (c-getsockname fd buf len) + (let ([p (sockaddr-in-port buf)]) + (foreign-free buf) + (foreign-free len) + p))]) + (make-tcp-server fd actual-port)))])) + + (define (tcp-server-port srv) + (tcp-server-port-num srv)) + + (define (tcp-accept srv) + ;; Accept a connection. Returns (values input-port output-port). + ;; Retries on EINTR (caused by GC stop-the-world interrupting accept()). + (let loop () + (let ([client-fd (c-accept (tcp-server-fd srv) 0 0)]) + (cond + [(>= client-fd 0) (fd->ports client-fd "tcp-client")] + [(= (get-errno) EINTR) (loop)] + [else (error 'tcp-accept "accept() failed")])))) + + (define (tcp-close srv) + (c-close (tcp-server-fd srv))) + + ;; ========== TCP Client ========== + + (define (tcp-connect address port) + ;; Connect to a TCP server. Returns (values input-port output-port). + ;; Retries on EINTR (caused by GC stop-the-world interrupting connect()). + (let ([fd (c-socket AF_INET SOCK_STREAM 0)]) + (when (< fd 0) + (error 'tcp-connect "socket() failed")) + (let ([addr (make-sockaddr-in address port)]) + (let loop () + (let ([rc (c-connect fd addr SOCKADDR_IN_SIZE)]) + (cond + [(>= rc 0) + (foreign-free addr) + (fd->ports fd "tcp-connection")] + [(= (get-errno) EINTR) (loop)] + [else + (foreign-free addr) + (c-close fd) + (error 'tcp-connect "connect() failed" address port)])))))) + + ;; ========== Convenience ========== + + (define-syntax with-tcp-server + (syntax-rules () + [(_ (var address port) body body* ...) + (let ([var (tcp-listen address port)]) + (dynamic-wind + (lambda () (void)) + (lambda () body body* ...) + (lambda () (tcp-close var))))])) + + ;; ========== Internal: FD → Ports ========== + + (define (fd->ports fd name) + ;; Wrap a socket FD as a pair of transcoded text ports. + ;; IMPORTANT: Both ports share the same fd. A closed? flag prevents: + ;; 1. Double-close: Chez calls close handler on both close-port AND GC finalization + ;; 2. Read/write after close: returns EOF/0 instead of operating on reused fd + (let ([closed? #f]) + (let ([in (make-custom-binary-input-port + (string-append name "-in") + (lambda (bv start count) + (if closed? 0 + (let ([buf (make-bytevector count)]) + (let retry () + (let ([n (c-read fd buf count)]) + (cond + [(> n 0) + (bytevector-copy! buf 0 bv start n) + n] + [(and (< n 0) (= (get-errno) EINTR)) (retry)] + [else 0])))))) + #f ;; get-position + #f ;; set-position! + (lambda () + (unless closed? + (set! closed? #t) + (c-close fd))))] + [out (make-custom-binary-output-port + (string-append name "-out") + (lambda (bv start count) + (if closed? 0 + (let ([buf (make-bytevector count)]) + (bytevector-copy! bv start buf 0 count) + (let lp ([written 0]) + (if (= written count) + count + (let ([n (c-write fd + (let ([tmp (make-bytevector (- count written))]) + (bytevector-copy! buf written tmp 0 (- count written)) + tmp) + (- count written))]) + (cond + [(> n 0) (lp (+ written n))]