Eliminate GC deadlock: merge master-timer into primordial thread
ober
17af2c27fb7a3e8029d3baac97a345bdd0c7ae2c
new file mode 100644 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,278 @@ +# Intermediate Docker image for jemacs-qt static builds. +# Bakes static Qt6 + QScintilla from source, plus all Chez Scheme +# dependencies (jerboa, gherkin, jsh, chez-pcre2, chez-scintilla, +# chez-qt) so that subsequent jemacs-qt builds only compile jemacs +# itself (~5-10 min instead of ~30 min). +# +# Build with: +# make docker-deps +# +# Uses BuildKit --build-context to pull dependency sources without +# copying them into the project directory. + +ARG ARCH=x86_64 +FROM alpine:3.21 + +# ── Phase 1: Alpine build deps ────────────────────────────────────────── +RUN apk add --no-cache \ + su-exec \ + cmake samurai perl python3 linux-headers patchelf \ + libxcb-dev xcb-util-dev xcb-util-image-dev \ + xcb-util-keysyms-dev xcb-util-renderutil-dev \ + xcb-util-wm-dev xcb-util-cursor-dev \ + libx11-dev libxkbcommon-dev \ + fontconfig-dev freetype-dev harfbuzz-dev \ + libpng-dev zlib-dev mesa-dev \ + pcre2-dev \ + at-spi2-core-dev libdrm-dev \ + zlib-static libxcb-static \ + fontconfig-static freetype-static harfbuzz-static \ + libpng-static bzip2-static expat-static brotli-static \ + libx11-static graphite2-static libxkbcommon-static \ + ncurses-dev ncurses-static \ + util-linux-dev util-linux-static \ + gcc g++ binutils make git curl wget + +# Build static libXau (no Alpine -static package available) +RUN apk add --no-cache libxau-dev && \ + cd /tmp && \ + wget -q https://xorg.freedesktop.org/releases/individual/lib/libXau-1.0.12.tar.xz && \ + tar xf libXau-1.0.12.tar.xz && \ + cd libXau-1.0.12 && \ + ./configure --prefix=/usr --enable-static --disable-shared && \ + make -j$(nproc) && make install && \ + cd / && rm -rf /tmp/libXau-1.0.12* + +# Build static libxcb-util (no Alpine -static package; needed by xcb-image) +RUN cd /tmp && \ + wget -q https://xcb.freedesktop.org/dist/xcb-util-0.4.1.tar.xz && \ + tar xf xcb-util-0.4.1.tar.xz && \ + cd xcb-util-0.4.1 && \ + ./configure --prefix=/usr --enable-static --disable-shared && \ + make -j$(nproc) && make install && \ + cd / && rm -rf /tmp/xcb-util-0.4.1* + +# ── Phase 2: Build Qt6 qtbase static ──────────────────────────────────── +ARG QT6_VERSION=6.8.3 +RUN wget -q https://download.qt.io/official_releases/qt/6.8/${QT6_VERSION}/submodules/qtbase-everywhere-src-${QT6_VERSION}.tar.xz && \ + tar xf qtbase-everywhere-src-${QT6_VERSION}.tar.xz && \ + rm qtbase-everywhere-src-${QT6_VERSION}.tar.xz && \ + cmake -S qtbase-everywhere-src-${QT6_VERSION} -B qt6-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/qt6-static \ + -DBUILD_SHARED_LIBS=OFF \ + -DQT_BUILD_EXAMPLES=OFF \ + -DQT_BUILD_TESTS=OFF \ + -DQT_BUILD_BENCHMARKS=OFF \ + -DFEATURE_xcb=ON \ + -DFEATURE_sql=OFF \ + -DFEATURE_network=OFF \ + -DFEATURE_testlib=OFF \ + -DFEATURE_printsupport=ON \ + -DFEATURE_dbus=OFF \ + -DFEATURE_opengl=OFF \ + -DFEATURE_vulkan=OFF \ + -DFEATURE_glib=OFF \ + -DFEATURE_icu=OFF && \ + cmake --build qt6-build --parallel && \ + cmake --install qt6-build && \ + rm -rf qtbase-everywhere-src-${QT6_VERSION} qt6-build + +# ── Phase 3: Build QScintilla static ──────────────────────────────────── +ARG QSCI_VERSION=2.14.1 +RUN wget -q https://www.riverbankcomputing.com/static/Downloads/QScintilla/${QSCI_VERSION}/QScintilla_src-${QSCI_VERSION}.tar.gz && \ + tar xf QScintilla_src-${QSCI_VERSION}.tar.gz && \ + rm QScintilla_src-${QSCI_VERSION}.tar.gz && \ + cd QScintilla_src-${QSCI_VERSION}/src && \ + /opt/qt6-static/bin/qmake CONFIG+=staticlib && \ + make -j$(nproc) && \ + make install && \ + cd / && rm -rf QScintilla_src-${QSCI_VERSION} + +# ── Phase 4: Generate pkg-config files with full transitive deps ───────── +# Static Qt6 cmake doesn't generate .pc files. We extract direct deps +# from .prl files, then add known transitive deps manually. +RUN mkdir -p /opt/qt6-static/lib/pkgconfig && \ + prl_libs() { \ + grep '^QMAKE_PRL_LIBS ' "$1" | \ + sed 's/^QMAKE_PRL_LIBS *= *//' | \ + tr ' ' '\n' | grep '^-l' | \ + grep -v '^-lQt6' | tr '\n' ' '; \ + } && \ + CORE_PRIVATE=$(prl_libs /opt/qt6-static/lib/libQt6Core.prl) && \ + GUI_PRL=$(prl_libs /opt/qt6-static/lib/libQt6Gui.prl) && \ + XCB_PRL=$(prl_libs /opt/qt6-static/plugins/platforms/libqxcb.prl) && \ + TRANSITIVE="-lgraphite2 -lbz2 -lbrotlidec -lbrotlicommon -lexpat -lXau -lXdmcp" && \ + GUI_PRIVATE="$GUI_PRL $TRANSITIVE" && \ + XCB_TRANSITIVE="-lxcb-util -lxcb -lXau -lXdmcp" && \ + XCB_PRIVATE="$XCB_PRL $TRANSITIVE $XCB_TRANSITIVE" && \ + echo "Core deps: $CORE_PRIVATE" && \ + echo "Gui deps: $GUI_PRIVATE" && \ + echo "XCB deps: $XCB_PRIVATE" && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + '' \ + 'Name: Qt6Core' 'Description: Qt6 Core' 'Version: 6.8.3' \ + 'Cflags: -I${includedir} -I${includedir}/QtCore' \ + "Libs: -L\${libdir} -lQt6Core" \ + "Libs.private: $CORE_PRIVATE" \ + > /opt/qt6-static/lib/pkgconfig/Qt6Core.pc && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + '' \ + 'Name: Qt6Gui' 'Description: Qt6 Gui' 'Version: 6.8.3' \ + 'Requires: Qt6Core' \ + 'Cflags: -I${includedir} -I${includedir}/QtGui' \ + "Libs: -L\${libdir} -lQt6Gui" \ + "Libs.private: $GUI_PRIVATE" \ + > /opt/qt6-static/lib/pkgconfig/Qt6Gui.pc && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + '' \ + 'Name: Qt6Widgets' 'Description: Qt6 Widgets' 'Version: 6.8.3' \ + 'Requires: Qt6Gui' \ + 'Cflags: -I${includedir} -I${includedir}/QtWidgets -I${includedir}/QtGui -I${includedir}/QtCore' \ + "Libs: -L\${libdir} -lQt6Widgets" \ + > /opt/qt6-static/lib/pkgconfig/Qt6Widgets.pc && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + '' \ + 'Name: Qt6PrintSupport' 'Description: Qt6 PrintSupport' 'Version: 6.8.3' \ + 'Requires: Qt6Widgets' \ + 'Cflags: -I${includedir} -I${includedir}/QtPrintSupport' \ + "Libs: -L\${libdir} -lQt6PrintSupport" \ + > /opt/qt6-static/lib/pkgconfig/Qt6PrintSupport.pc && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + 'plugindir=${prefix}/plugins' \ + '' \ + 'Name: Qt6XcbPlugin' 'Description: Qt6 XCB platform plugin' 'Version: 6.8.3' \ + 'Requires: Qt6Gui' \ + "Libs: -L\${libdir} -L\${plugindir}/platforms -lqxcb -lQt6XcbQpa" \ + "Libs.private: $XCB_PRIVATE" \ + > /opt/qt6-static/lib/pkgconfig/Qt6XcbPlugin.pc && \ + printf '%s\n' \ + 'prefix=/opt/qt6-static' \ + 'includedir=${prefix}/include' \ + 'libdir=${prefix}/lib' \ + '' \ + 'Name: QScintilla' 'Description: QScintilla for Qt6' 'Version: 2.14.1' \ + 'Requires: Qt6Widgets Qt6PrintSupport' \ + 'Cflags: -I${includedir} -I${includedir}/Qsci' \ + "Libs: -L\${libdir} -lqscintilla2_qt6" \ + > /opt/qt6-static/lib/pkgconfig/QScintilla.pc && \ + echo "Generated .pc files:" && ls /opt/qt6-static/lib/pkgconfig/ + +# ── Phase 5: Build Chez Scheme from source for musl ───────────────────── +# Build WITHOUT --static so libkernel.a retains full dlopen support. +# This is needed because jemacs embeds the program as a .so and loads it +# at runtime via Sscheme_script (which calls dlopen internally). +# musl's static libdl.a provides dlopen in the final static binary. +ARG CHEZ_TAG=main +RUN git clone --depth 1 --branch ${CHEZ_TAG} \ + https://github.com/cisco/ChezScheme /tmp/ChezScheme && \ + cd /tmp/ChezScheme && \ + ./configure --threads --installprefix=/opt/chez && \ + make -j$(nproc) && \ + make install && \ + rm -rf /tmp/ChezScheme + +# ── Phase 6: Gerbil/Chez dependencies ────────────────────────────────── +ENV PKG_CONFIG_PATH=/opt/qt6-static/lib/pkgconfig +ENV SCHEME=/opt/chez/bin/scheme + +# Copy dependency sources from build contexts +COPY --from=jerboa-src . /deps/jerboa +COPY --from=gherkin-src . /deps/gherkin +COPY --from=jsh-src . /deps/jsh +COPY --from=pcre2-src . /deps/chez-pcre2 +COPY --from=sci-src . /deps/chez-scintilla +COPY --from=qt-src . /deps/chez-qt +COPY --from=qtshim-src . /deps/gerbil-qt + +# Pre-compile all Chez library dependencies. +# These .so files are baked into the image so jemacs builds only +# need to compile jemacs-specific modules. +RUN /opt/chez/bin/scheme \ + --libdirs /deps/jerboa/lib:/deps/gherkin/src:/deps/jsh/src:/deps/chez-pcre2:/deps/chez-scintilla/src:/deps/chez-qt \ + --compile-imported-libraries \ + --script /dev/stdin <<'EOF' +#!chezscheme +(import + (except (chezscheme) make-hash-table hash-table? iota 1+ 1- + getenv path-extension path-absolute? thread? + make-mutex mutex? mutex-name) + (jerboa core) + (jerboa runtime) + (std sugar) + (std format) + (std sort) + (std pregexp) + (std foreign) + (std misc list) + (std misc thread) + (std os path) + (std os signal) + (compat types) + (compat gambit-compat) + (runtime util) + (runtime table) + (runtime c3) + (runtime mop) + (compat gambit) + (jsh ffi) + (jsh static-compat) + (chez-scintilla constants) + (chez-qt ffi) + (chez-qt qt)) +(display "Chez deps pre-compiled OK\n") +EOF + +# Build static pcre2 shim object +RUN PCRE2_CFLAGS=$(pkg-config --cflags libpcre2-8 2>/dev/null || echo "") && \ + gcc -c -O2 -o /deps/chez-pcre2/pcre2_shim.o \ + /deps/chez-pcre2/pcre2_shim.c $PCRE2_CFLAGS -Wall + +# Build static jsh FFI shim object +RUN gcc -c -O2 -o /deps/jsh/jsh_ffi_shim.o /deps/jsh/ffi-shim.c -Wall + +# Build static qt_chez_shim object +RUN QT_CFLAGS=$(pkg-config --cflags Qt6Widgets 2>/dev/null || \ + echo "-I/opt/qt6-static/include -I/opt/qt6-static/include/QtCore \ + -I/opt/qt6-static/include/QtGui -I/opt/qt6-static/include/QtWidgets") && \ + gcc -c -O2 -fPIC -o /deps/chez-qt/qt_chez_shim.o \ + /deps/chez-qt/qt_chez_shim.c \ + -I/deps/gerbil-qt/vendor $QT_CFLAGS -Wall + +# Build static libqt_shim.a from qt_shim.cpp +# qt_static_plugins.o must be separate (NOT in archive) — it contains +# Q_IMPORT_PLUGIN static constructors that the linker drops from archives. +RUN QT_CFLAGS=$(pkg-config --cflags Qt6Widgets 2>/dev/null || \ + echo "-I/opt/qt6-static/include -I/opt/qt6-static/include/QtCore \ + -I/opt/qt6-static/include/QtGui -I/opt/qt6-static/include/QtWidgets") && \ + QSCI_FLAGS="-DQT_SCINTILLA_AVAILABLE \ + $(pkg-config --cflags QScintilla 2>/dev/null || \ + echo "-I/opt/qt6-static/include -I/opt/qt6-static/include/Qsci")" && \ + g++ -c -fPIC -std=c++17 \ + $QT_CFLAGS $QSCI_FLAGS \ + /deps/gerbil-qt/vendor/qt_shim.cpp \ + -o /deps/gerbil-qt/vendor/qt_shim_static.o && \ + ar rcs /deps/gerbil-qt/vendor/libqt_shim.a \ + /deps/gerbil-qt/vendor/qt_shim_static.o && \ + printf '#include <QtPlugin>\nQ_IMPORT_PLUGIN(QXcbIntegrationPlugin)\n' \ + > /deps/gerbil-qt/vendor/qt_static_plugins.cpp && \ + g++ -c -fPIC -std=c++17 $QT_CFLAGS \ + /deps/gerbil-qt/vendor/qt_static_plugins.cpp \ + -o /deps/gerbil-qt/vendor/qt_static_plugins.o + +WORKDIR /src --- a/Makefile +++ b/Makefile @@ -269,6 +269,10 @@ build-jemacs-qt-static: check-root JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /deps/jerboa/lib \ --compile-imported-libraries --script /src/vendor/jerboa-compile-tcp.ss && \ rm -f /deps/jerboa/lib/std/net/*.wpo && \ + rm -f /src/lib/jerboa/*.wpo /src/lib/jerboa/*.so && \ + JEMACS_STATIC=1 /opt/chez/bin/scheme --libdirs /src/lib \ + --compile-imported-libraries --script /src/vendor/jerboa-compile-repl-socket.ss && \ + rm -f /src/lib/jerboa/*.wpo && \ JEMACS_STATIC=1 \ CHEZ_DIR=$(CHEZ_MUSL_DIR) \ JERBOA_DIR=/deps/jerboa/lib \ new file mode 100644 --- /dev/null +++ b/build-binary-qt.ss @@ -0,0 +1,499 @@ +#!chezscheme +;; Build a native jemacs-qt binary for jerboa-emacs (Qt frontend). +;; +;; Usage: cd jerboa-emacs && make binary-qt +;; +;; Produces: ./jemacs-qt (single ELF binary with embedded boot files + program) +;; +;; All boot files are embedded as C byte arrays via Sregister_boot_file_bytes. +;; FFI shims are compiled into the binary (pcre2_shim, qt_chez_shim). +;; libqt_shim.so (from gerbil-qt vendor) is loaded at runtime — it must +;; be in the same directory as the binary or on LD_LIBRARY_PATH. +;; +;; Qt system libraries (Qt6Widgets, Qt6Core, etc.) are linked dynamically +;; from the system install. The binary requires Qt6 to be installed. + +(import (chezscheme)) + +;; --- Helper: generate C header from binary file --- +(define (file->c-header input-path output-path array-name size-name) + (let* ((port (open-file-input-port input-path)) + (data (get-bytevector-all port)) + (size (bytevector-length data))) + (close-port port) + (call-with-output-file output-path + (lambda (out) + (fprintf out "/* Auto-generated — do not edit */~n") + (fprintf out "static const unsigned char ~a[] = {~n" array-name) + (let loop ((i 0)) + (when (< i size) + (when (= 0 (modulo i 16)) (fprintf out " ")) + (fprintf out "0x~2,'0x" (bytevector-u8-ref data i)) + (when (< (+ i 1) size) (fprintf out ",")) + (when (= 15 (modulo i 16)) (fprintf out "~n")) + (loop (+ i 1)))) + (fprintf out "~n};~n") + (fprintf out "static const unsigned int ~a = ~a;~n" size-name size)) + 'replace) + (printf " ~a: ~a bytes~n" output-path size))) + +;; --- Locate Chez install directory --- +(define chez-dir + (or (getenv "CHEZ_DIR") + (let* ((mt (symbol->string (machine-type))) + (home (getenv "HOME")) + (lib-dir (format "~a/.local/lib" home)) + (csv-dir + (let lp ((dirs (guard (e (#t '())) (directory-list lib-dir)))) + (cond + ((null? dirs) #f) + ((and (> (string-length (car dirs)) 3) + (string=? "csv" (substring (car dirs) 0 3))) + (format "~a/~a/~a" lib-dir (car dirs) mt)) + (else (lp (cdr dirs))))))) + (and csv-dir + (file-exists? (format "~a/main.o" csv-dir)) + csv-dir)))) + +(unless chez-dir + (display "Error: Cannot find Chez install dir. Set CHEZ_DIR.\n") + (exit 1)) + +;; --- Locate dependent library directories --- +(define home (getenv "HOME")) +(define jerboa-dir + (or (getenv "JERBOA_DIR") + (format "~a/mine/jerboa/lib" home))) +(define gherkin-dir + (or (getenv "GHERKIN_DIR") + (format "~a/mine/gherkin/src" home))) +(define jsh-dir + (or (getenv "JSH_DIR") + (format "~a/mine/jerboa-shell/src" home))) +(define pcre2-dir + (or (getenv "CHEZ_PCRE2_DIR") + (format "~a/mine/chez-pcre2" home))) +(define sci-dir + (or (getenv "CHEZ_SCINTILLA_DIR") + (format "~a/mine/chez-scintilla/src" home))) +(define qt-dir + (or (getenv "CHEZ_QT_DIR") + (format "~a/mine/chez-qt" home))) +(define qt-shim-dir + (or (getenv "CHEZ_QT_SHIM_DIR") + (format "~a/mine/gerbil-qt/vendor" home))) + +;; Static build detection (needed before dep checks) +(define jemacs-static? + (let ((v (getenv "JEMACS_STATIC"))) + (and v (not (string=? v "0")) (not (string=? v ""))))) + +;; Check critical dependencies +(for-each + (lambda (path label) + (unless (file-exists? path) + (printf "Error: ~a not found at ~a~n" label path) + (exit 1))) + (list (format "~a/jerboa/core.so" jerboa-dir) + (if jemacs-static? + (format "~a/qt_chez_shim.o" qt-dir) + (format "~a/qt_chez_shim.so" qt-dir)) + (if jemacs-static? + (format "~a/pcre2_shim.o" pcre2-dir) + (format "~a/pcre2_shim.so" pcre2-dir)) + (if jemacs-static? + (format "~a/libqt_shim.a" qt-shim-dir) + (format "~a/libqt_shim.so" qt-shim-dir))) + (list "jerboa core.so" + (if jemacs-static? "qt_chez_shim.o" "qt_chez_shim.so") + (if jemacs-static? "pcre2_shim.o" "pcre2_shim.so") + (if jemacs-static? "libqt_shim.a" "libqt_shim.so"))) + +(printf "Chez dir: ~a~n" chez-dir) +(printf "Jerboa dir: ~a~n" jerboa-dir) +(printf "Gherkin dir: ~a~n" gherkin-dir) +(printf "jsh dir: ~a~n" jsh-dir) +(printf "pcre2 dir: ~a~n" pcre2-dir) +(printf "Sci dir: ~a~n" sci-dir) +(printf "Qt dir: ~a~n" qt-dir) +(printf "Qt shim dir: ~a~n" qt-shim-dir) + +;; --- Step 1: Compile all modules + entry point --- +(printf "~n[1/7] Compiling all modules (optimize-level 3, WPO)...~n") +(parameterize ((compile-imported-libraries #t) + (optimize-level 3) + (cp0-effort-limit 500) + (cp0-score-limit 50) + (cp0-outer-unroll-limit 1) + (commonization-level 4) + (enable-unsafe-application #t) + (enable-unsafe-variable-reference #t) + (enable-arithmetic-left-associative #t) + (debug-level 0) + (generate-inspector-information #f) + (generate-wpo-files #t)) + (compile-program "qt-main.ss")) + +;; --- Step 2: Whole-program optimization --- +(printf "[2/7] Running whole-program optimization...~n") +(let ((missing (compile-whole-program "qt-main.wpo" "jemacs-qt-all.so"))) + (unless (null? missing) + (printf " WPO: ~a libraries not incorporated (missing .wpo):~n" (length missing)) + (for-each (lambda (lib) (printf " ~a~n" lib)) missing))) + +;; --- Step 3: Make libs-only boot file --- +(printf "[3/7] Creating libs-only boot file...~n") +(define (existing-so-files paths) + (filter file-exists? paths)) +(apply make-boot-file "jemacs-qt.boot" '("scheme" "petite") + (existing-so-files + (append + ;; Jerboa runtime (no deps on other jerboa libs) + (map (lambda (m) (format "~a/~a.so" jerboa-dir m)) + '("jerboa/runtime")) + ;; Jerboa stdlib + (map (lambda (m) (format "~a/~a.so" jerboa-dir m)) + '("std/error" + "std/format" + "std/sort" + "std/pregexp" + "std/foreign" + "std/misc/string" + "std/misc/list" + "std/misc/alist" + "std/misc/thread" + "std/os/path" + "std/os/signal" + "std/os/fdio")) + ;; Jerboa core + sugar + (map (lambda (m) (format "~a/~a.so" jerboa-dir m)) + '("jerboa/core" + "std/sugar")) + ;; std/net/tcp (compiled by step 1) + (list (format "~a/std/net/tcp.so" jerboa-dir)) + ;; jerboa/repl-socket (non-blocking socket FFI for debug REPL + IPC) + (list "lib/jerboa/repl-socket.so") + ;; Gherkin MOP modules (WPO-missing, must be in boot file) + (map (lambda (m) (format "~a/~a.so" gherkin-dir m)) + '("compat/types" + "compat/gambit-compat" + "runtime/util" + "runtime/table" + "runtime/c3" + "runtime/mop")) + ;; compat/gambit lives in jsh (not gherkin) + (list (format "~a/compat/gambit.so" jsh-dir)) + ;; jsh modules (WPO-missing) + (map (lambda (m) (format "~a/jsh/~a.so" jsh-dir m)) + '("embed" + "embed-data" + "arithmetic" + "functions" + "ast" + "environment" + "registry" + "ffi" + "static-compat")) + ;; chez-pcre2 (compiled by step 1) + (map (lambda (m) (format "~a/chez-pcre2/~a.so" pcre2-dir m)) + '("ffi" "pcre2")) + ;; chez-scintilla: ffi (patched to skip load-shared-object in static builds) + constants + (map (lambda (m) (format "~a/chez-scintilla/~a.so" sci-dir m)) + '("ffi" "constants")) + ;; chez-qt + (map (lambda (m) (format "~a/chez-qt/~a.so" qt-dir m)) + '("ffi" "qt")) + ;; jerboa-emacs shared modules (topological order) + (map (lambda (m) (format "lib/jerboa-emacs/~a.so" m)) + '(;; Tier 0: no jerboa-emacs deps + "customize" "themes" "pregexp-compat" "macros" + "vtscreen" "pty" "ipc" "debug-repl" "snippets" + ;; Tier 1: face + "face" + ;; Tier 2: core + "core" + ;; Tier 3: base editor modules + "buffer" "echo" "keymap" "repl" "shell-history" + "subprocess" "gsh-subprocess" "async" "chat" + "highlight" + ;; Tier 4: window system + "window" "modeline" "persist" + ;; Tier 5: org + shell + "org-parse" "gsh-eshell" "eshell" "shell" + "org-export" "org-highlight" "org-list" "org-table" + "org-clock" "org-babel" "org-capture" "org-agenda" + ;; Tier 6: helm + terminal + "helm" "terminal" + "helm-sources" "helm-tui" + ;; Tier 7: helm commands + editor extras + "helm-commands" "editor-extra-helpers" + ;; Tier 8: editor core + "editor-core" + ;; Tier 9: editor UI + "editor-ui" "editor-text" + ;; Tier 10: advanced editor + "editor-advanced" + ;; Tier 11-13: editor commands + "editor-cmds-a" "editor-cmds-b" "editor-cmds-c" + ;; Tier 14+: editor extras chain + "editor-extra-ai" + "editor-extra-editing" "editor-extra-editing2" + "editor-extra-media" "editor-extra-media2" + "editor-extra-modes" + "editor-extra-org" + "editor-extra-regs" "editor-extra-regs2" + "editor-extra-tools" "editor-extra-tools2" + "editor-extra-vcs" "editor-extra-web" + "editor-extra-final" "editor-extra" + ;; TUI top-level (editor shared with Qt) + "editor")) + ;; Qt-specific modules + (map (lambda (m) (format "lib/jerboa-emacs/qt/~a.so" m)) + '(;; Foundation + "sci-shim" "keymap" "buffer" "echo" + "image" "magit" + "highlight" "modeline" "window" + "lsp-client" "helm-qt" + "snippets" "menubar" + ;; Commands (pairs: base + extended) + "commands-core" "commands-core2" + "commands-edit" "commands-edit2" + "commands-file" "commands-file2" + "commands-search" "commands-search2" + "commands-sexp" "commands-sexp2" + "commands-shell" "commands-shell2" + "commands-vcs" "commands-vcs2" + "commands-ide" "commands-ide2" + "commands-lsp" + "commands-modes" "commands-modes2" + "commands-parity" "commands-parity2" + "commands-parity3" "commands-parity3b" + "commands-parity4" "commands-parity5" + "commands-config" "commands-config2" + "commands-aliases" "commands-aliases2" + "commands" + ;; Qt app + "app" "main"))))) + +;; --- Step 4: Generate C headers with embedded data --- +(printf "[4/7] Embedding boot files + program as C headers...~n") +(file->c-header "jemacs-qt-all.so" "jemacs_qt_program.h" + "jemacs_qt_program_data" "jemacs_qt_program_size") +(file->c-header (format "~a/petite.boot" chez-dir) "jemacs_qt_petite_boot.h" + "petite_boot_data" "petite_boot_size") +(file->c-header (format "~a/scheme.boot" chez-dir) "jemacs_qt_scheme_boot.h" + "scheme_boot_data" "scheme_boot_size") +(file->c-header "jemacs-qt.boot" "jemacs_qt_jemacs_qt_boot.h" + "jemacs_qt_boot_data" "jemacs_qt_boot_size") + +;; --- Helper: run a shell command and capture its first output line --- +(define (shell-output cmd default) + (let ((tmpfile "/tmp/jemacs-qt-build-tmp.txt")) + (system (format "~a > ~a 2>/dev/null; true" cmd tmpfile)) + (if (file-exists? tmpfile) + (let* ((p (open-input-file tmpfile)) + (line (get-line p))) + (close-port p) + (delete-file tmpfile) + (if (eof-object? line) default line)) + default))) + +(when jemacs-static? + (printf "Static build mode: JEMACS_STATIC=1~n")) + +;; --- Step 5: Compile C sources --- +(printf "[5/7] Compiling C sources...~n") + +;; pcre2 shim +(let* ((pcre2-cflags (shell-output "pkg-config --cflags libpcre2-8" "-I/usr/include")) + (cmd (format "gcc -c -O2 -o jemacs-qt-pcre2-shim.o ~a/pcre2_shim.c ~a -Wall 2>&1" + pcre2-dir pcre2-cflags))) + (unless (= 0 (system cmd)) + (display "Error: pcre2 shim compilation failed\n") + (exit 1))) + +;; qt_chez_shim +(let* ((qt-cflags (shell-output "pkg-config --cflags Qt6Widgets" "")) + (cmd (format "gcc -c -O2 -DQT_SCINTILLA_AVAILABLE -o jemacs-qt-chez-shim.o ~a/qt_chez_shim.c -I~a ~a -Wall 2>&1" + qt-dir qt-shim-dir qt-cflags))) + (unless (= 0 (system cmd)) + (display "Error: qt_chez_shim compilation failed\n") + (exit 1))) + +;; Static: generate + compile foreign symbol registration table +;; (dlopen(NULL) is a stub in musl static builds, so we use Sforeign_symbol instead) +(when jemacs-static? + ;; scheme.h lives in chez-dir (same dir as libkernel.a) + (let* ((ffi-path (format "~a/chez-qt/ffi.ss" qt-dir)) + (pcre2-ffi-path (format "~a/chez-pcre2/ffi.ss" pcre2-dir)) + (include-dir chez-dir) + ;; Generate the C file by scanning ALL FFI source files: + ;; chez-qt/ffi.ss, chez-pcre2/ffi.ss — specific files + ;; jsh-dir/, jerboa-dir/, lib/jerboa-emacs/ — recursive (catches all transitive deps) + ;; Strip Scheme line comments (;; ...) before matching to avoid placeholder strings + ;; like "c_name", "c_func", "c_function_name" that appear in macro doc comments. + ;; Two patterns extracted: + ;; (foreign-procedure "name") — direct Chez FFI + ;; define-foreign name "c-name" — jsh macro (C name is the second string) + (gen-cmd + (format + "{ { cat ~a ~a; find ~a ~a ~a lib/jerboa-emacs -name '*.sls' -o -name '*.ss' | xargs cat 2>/dev/null; } | \ +sed 's/;;.*//' | grep -oE '(foreign-procedure|foreign-entry\\?) \"[^\"]*\"' | sed 's/.* \"//;s/\"//'; \ +{ cat ~a ~a; } | sed 's/;;.*//' | grep -o 'define-optional-ffi [^ ]* \"[^\"]*\"' | sed 's/.*define-optional-ffi [^ ]* \"//;s/\"//'; \ +find ~a -name '*.sls' -o -name '*.ss' | \ + xargs grep -oh 'define-foreign [^ ]* \"[^\"]*\"' 2>/dev/null | \ + sed 's/.*define-foreign [^ ]* \"//;s/\".*//'; } | \ +sort -u | grep -v '^$' | grep -v '^_NSGetExecutablePath$' | grep -v '^io_uring_' > /tmp/ffi_syms.txt && \ +awk '\ +BEGIN{ print \"/* Auto-generated — do not edit */\"; \ + print \"#include \\\"scheme.h\\\"\"; print \"\"; } \ +{ print \"extern void \" $1 \"(void);\" } \ +END{ print \"\"; print \"void register_static_foreign_symbols(void) {\"; } \ +' /tmp/ffi_syms.txt > qt_static_symbols.c && \ +awk '{ print \" Sforeign_symbol(\\\"\" $1 \"\\\", (void*)\" $1 \");\" }' \ +/tmp/ffi_syms.txt >> qt_static_symbols.c && \ +echo \"}\" >> qt_static_symbols.c && \ +rm /tmp/ffi_syms.txt && \ +echo OK" + ffi-path pcre2-ffi-path + jsh-dir jerboa-dir sci-dir + ffi-path pcre2-ffi-path + jsh-dir)) + (result (shell-output gen-cmd ""))) + (unless (string=? (string-downcase (substring result 0 (min 2 (string-length result)))) "ok") + (printf "Error generating qt_static_symbols.c: ~a~n" result) + (exit 1)) + (let* ((count-cmd "grep -c 'Sforeign_symbol' qt_static_symbols.c") + (count (let ((s (shell-output count-cmd "0"))) + ;; trim trailing newline + (if (and (> (string-length s) 0) + (char=? (string-ref s (- (string-length s) 1)) #\newline)) + (substring s 0 (- (string-length s) 1)) + s)))) + (printf " Generated qt_static_symbols.c with ~a symbol registrations~n" count)) + (let* ((cmd (format "gcc -c -O2 -o qt_static_symbols.o qt_static_symbols.c -I~a -Wall 2>&1" + include-dir))) + (unless (= 0 (system cmd)) + (display "Error: qt_static_symbols.c compilation failed\n") + (exit 1))))) + +;; jsh FFI shim (needed for static builds so ffi_* symbols are in the binary) +;; ffi-shim.c lives in the jsh root (parent of jsh-dir which is the src/ subdir) +(when jemacs-static? + (let* ((jsh-root (path-parent jsh-dir)) + (cmd (format "gcc -c -O2 -o jemacs-qt-jsh-ffi.o ~a/ffi-shim.c -Wall 2>&1" + jsh-root))) + (unless (= 0 (system cmd)) + (display "Error: jsh ffi-shim.c compilation failed\n") + (exit 1)))) + +;; chez-scintilla stubs (TUI-only — Qt never calls these; stubs allow foreign-procedure defs) +(when jemacs-static? + (let* ((cmd "gcc -c -O2 -o jemacs-qt-sci-stubs.o support/chez_scintilla_stubs.c -Wall 2>&1")) + (unless (= 0 (system cmd)) + (display "Error: chez_scintilla_stubs.c compilation failed\n") + (exit 1)))) + +;; pty shim (needed for static builds — pty_* symbols from support/pty_shim.c) +(when jemacs-static? + (let* ((cmd "gcc -c -O2 -o jemacs-qt-pty-shim.o support/pty_shim.c -Wall 2>&1")) + (unless (= 0 (system cmd)) + (display "Error: pty_shim.c compilation failed\n") + (exit 1)))) + +;; jerboa landlock shim (jerboa_landlock_* symbols from jerboa/support/landlock-shim.c) +(when jemacs-static? + (let* ((jerboa-root (path-parent jerboa-dir)) + (cmd (format "gcc -c -O2 -o jemacs-qt-jerboa-landlock.o ~a/support/landlock-shim.c -Wall 2>&1" + jerboa-root))) + (unless (= 0 (system cmd)) + (display "Error: landlock-shim.c compilation failed\n") + (exit 1)))) + +;; jemacs-qt-main.c +(let* ((static-flag (if jemacs-static? "-DJEMACS_STATIC_BUILD" "")) + (cmd (format "gcc -c -O2 ~a -o jemacs-qt-main.o jemacs-qt-main.c -I~a -I. -Wall 2>&1" + static-flag chez-dir))) + (unless (= 0 (system cmd)) + (display "Error: jemacs-qt-main.c compilation failed\n") + (exit 1))) + +;; --- Step 6: Link native binary --- +(printf "[6/7] Linking native binary...~n") +(if jemacs-static? + ;; ─── Static link (Docker musl build) ─────────────────────────────────── + ;; -static: fully static binary (musl embeds dynamic linker, dlopen works) + ;; -Wl,--export-dynamic: export main binary symbols so the embedded .so + ;; (loaded via Sscheme_script/dlopen from memfd) can find Chez kernel symbols + ;; qt_static_plugins.o must be linked OUTSIDE archives (not in libqt_shim.a) + ;; because it contains Q_IMPORT_PLUGIN static constructors the linker drops from archives + (let* ((pcre2-libs (shell-output "pkg-config --static --libs libpcre2-8" "-lpcre2-8")) + (qt-libs (shell-output + "pkg-config --static --libs Qt6Widgets Qt6XcbPlugin QScintilla" + "-lQt6Widgets -lQt6Gui -lQt6Core")) + (qt-plugins (format "~a/qt_static_plugins.o" qt-shim-dir)) + (libqt-shim (format "~a/libqt_shim.a" qt-shim-dir)) + (cmd (format "g++ -static -Wl,--export-dynamic -o jemacs-qt \ +jemacs-qt-main.o jemacs-qt-chez-shim.o jemacs-qt-pcre2-shim.o jemacs-qt-jsh-ffi.o \ +jemacs-qt-pty-shim.o jemacs-qt-jerboa-landlock.o jemacs-qt-sci-stubs.o \ +qt_static_symbols.o \ +~a ~a ~a ~a \ +-L~a -lkernel -llz4 -lz \ +-lm -ldl -lpthread -luuid -lncurses -lstdc++ 2>&1" + libqt-shim qt-plugins qt-libs pcre2-libs + chez-dir))) + (printf " ~a~n" cmd) + (unless (= 0 (system cmd)) + (display "Error: Static link failed\n") + (exit 1))) + ;; ─── Dynamic link (default local build) ──────────────────────────────── + (let* ((pcre2-libs (shell-output "pkg-config --libs libpcre2-8" "-lpcre2-8")) + (qt-libs (shell-output "pkg-config --libs Qt6Widgets" "-lQt6Widgets -lQt6Gui -lQt6Core")) + (cmd (format "g++ -rdynamic -o jemacs-qt jemacs-qt-main.o jemacs-qt-chez-shim.o jemacs-qt-pcre2-shim.o ~a ~a -L~a -lkernel -llz4 -lz -lm -ldl -lpthread -luuid -lncurses -lstdc++ -L~a -lqt_shim -lqscintilla2_qt6 -Wl,-rpath,~a -Wl,-rpath,~a 2>&1" + pcre2-libs qt-libs chez-dir qt-shim-dir chez-dir qt-shim-dir))) + (printf " ~a~n" cmd) + (unless (= 0 (system cmd)) + (display "Error: Link failed\n") + (exit 1)))) + +;; --- Step 7: Clean up intermediate files --- +(printf "[7/7] Cleaning up...~n") +(for-each (lambda (f) + (when (file-exists? f) (delete-file f))) + (append + '("jemacs-qt-main.o" "jemacs-qt-chez-shim.o" "jemacs-qt-pcre2-shim.o" + "jemacs_qt_program.h" "jemacs_qt_petite_boot.h" + "jemacs_qt_scheme_boot.h" "jemacs_qt_jemacs_qt_boot.h" + "jemacs-qt-all.so" "qt-main.so" "qt-main.wpo" "jemacs-qt.boot") + (if jemacs-static? + '("jemacs-qt-jsh-ffi.o" "jemacs-qt-pty-shim.o" "jemacs-qt-jerboa-landlock.o" + "jemacs-qt-sci-stubs.o" "qt_static_symbols.o" "qt_static_symbols.c") + '()))) + +(printf "~n========================================~n") +(printf "Build complete!~n~n") +(printf " Binary: ./jemacs-qt (~a KB)~n" + (quotient (file-length (open-file-input-port "jemacs-qt")) 1024)) +(if jemacs-static? + (begin + (printf "~nStatically linked — no runtime .so dependencies.~n") + (printf "~nRun:~n") + (printf " ./jemacs-qt # launch Qt editor~n") + (printf " ./jemacs-qt file.txt # open file~n") + (printf "~nInstall:~n") + (printf " cp jemacs-qt /usr/local/bin/~n")) + (begin + ;; Copy shim .so files alongside binary for dynamic builds + (system (format "cp ~a/libqt_shim.so . 2>/dev/null; true" qt-shim-dir)) + (system (format "cp ~a/qt_chez_shim.so . 2>/dev/null; true" qt-dir)) + (system (format "cp ~a/pcre2_shim.so . 2>/dev/null; true" pcre2-dir)) + (printf "~nBundle (keep these together):~n") + (printf " ./jemacs-qt~n") + (printf " ./libqt_shim.so~n") + (printf " ./qt_chez_shim.so~n") + (printf " ./pcre2_shim.so~n") + (printf "~nRun:~n") + (printf " ./jemacs-qt # launch Qt editor~n") + (printf " ./jemacs-qt file.txt # open file~n") + (printf "~nInstall:~n") + (printf " cp jemacs-qt libqt_shim.so qt_chez_shim.so pcre2_shim.so /usr/local/bin/~n"))) --- a/lib/jerboa-emacs/qt/app.sls +++ b/lib/jerboa-emacs/qt/app.sls @@ -132,6 +132,7 @@ (qt-plain-text-edit-set-selection! ed mark pos)) (let ([pos (qt-plain-text-edit-cursor-position ed)]) (qt-plain-text-edit-set-selection! ed pos pos))))) + (def *master-timer-tick-fn* #f) (def *which-key-timer* #f) (def *which-key-pending-keymap* #f) (def *which-key-pending-prefix* #f) @@ -1545,20 +1546,17 @@ (for-each (lambda (f) (qt-open-file! app f)) (ipc-poll-files!)))) - (spawn/name/pinned - 'master-timer + (set! *master-timer-tick-fn* (lambda () - (let loop () - (thread-sleep! 0.05) - (qt-drain-pending-callbacks!) - (master-timer-tick!) - (loop)))))) + (qt-drain-pending-callbacks!) + (master-timer-tick!))))) (def (qt-main . args) (pin-thread-to-processor0! (current-thread)) (setenv "QT_IM_MODULE" "compose") (setenv "QT_ACCESSIBILITY" "0") (let ([qt-app (qt-app-create)]) - (try (qt-do-init! qt-app args) (qt-app-exec! qt-app) (lsp-stop!) + (try (qt-do-init! qt-app args) + (qt-app-exec! qt-app *master-timer-tick-fn*) (lsp-stop!) (stop-ipc-server!) (stop-debug-repl!) (finally (qt-app-quit! qt-app) (qt-app-destroy! qt-app))))) (def (qt-open-file! app filename (on-loaded #f)) --- a/src/jerboa-emacs/qt/app.ss +++ b/src/jerboa-emacs/qt/app.ss @@ -143,6 +143,9 @@ (let ((pos (qt-plain-text-edit-cursor-position ed))) (qt-plain-text-edit-set-selection! ed pos pos))))) +;; Master timer tick function — set by qt-do-init!, called from qt-app-exec! loop +(def *master-timer-tick-fn* #f) + ;; Which-key state — show available bindings after prefix key delay (def *which-key-timer* #f) (def *which-key-pending-keymap* #f) @@ -1198,32 +1201,14 @@ (for-each (lambda (f) (qt-open-file! app f)) (ipc-poll-files!)))) - ;; Master timer — drives all periodic tasks and drains the async UI queue. - ;; Single 50ms green-thread loop replaces 7+ individual Qt timers. - ;; - ;; SMP NOTE: We deliberately use a spawned Gambit green thread instead of - ;; a Qt QTimer. In the SMP model the Qt thread is a raw pthread (not a - ;; Gambit VP), so any `c-define` callback invoked directly from the Qt - ;; thread has ___ps == NULL and crashes. A Qt timer's timeout signal fires - ;; on the Qt thread, so its trampoline would enqueue the callback instead - ;; of executing it — creating a circular deadlock where the drain is inside - ;; the callback that never runs. - ;; - ;; A spawned Gambit green thread solves this cleanly: thread-sleep! yields - ;; to the Gambit scheduler (no Qt calls), and the tick/drain run on a - ;; Gambit VP where ___ps is valid. Qt calls inside the drain/tick go - ;; through BlockingQueuedConnection to the Qt thread as normal. - ;; Pin the master timer to processor 0 so it always drains the BQC - ;; callback queue and runs periodic UI tasks on the main OS thread. - ;; Without pinning, work-stealing could move it to another VP where - ;; Qt calls via BQC would still work but add unnecessary latency. - (spawn/name/pinned 'master-timer + ;; Master timer tick function — passed to qt-app-exec! as the per-iteration + ;; callback so all periodic work runs on the primordial thread. + ;; This eliminates the second Chez thread entirely, preventing GC + ;; rendezvous deadlocks (single thread = no stop-the-world coordination). + (set! *master-timer-tick-fn* (lambda () - (let loop () - (thread-sleep! 0.05) - (qt-drain-pending-callbacks!) - (master-timer-tick!) - (loop)))) + (qt-drain-pending-callbacks!) + (master-timer-tick!))) )) ;; end of qt-do-init! let* and function body @@ -1250,8 +1235,11 @@ ;; background threads (LSP, async file I/O) use BlockingQueuedConnection ;; safely because the event loop is then running. (qt-do-init! qt-app args) - ;; Enter Qt event loop (blocks here until quit) - (qt-app-exec! qt-app) + ;; Enter Qt event loop (blocks here until quit). + ;; Pass the master-timer tick as the per-iteration callback so all + ;; periodic work (UI queue drain, scheduled tasks, debug REPL, IPC) + ;; runs on the primordial thread — single Chez thread, no GC deadlock. + (qt-app-exec! qt-app *master-timer-tick-fn*) ;; Cleanup after event loop exits (lsp-stop!) (stop-ipc-server!) --- a/tests/test-debug-repl.ss +++ b/tests/test-debug-repl.ss @@ -4,7 +4,9 @@ (import (except (chezscheme) make-hash-table hash-table? iota 1+ 1-) + (only (jerboa core) thread-terminate!) (jerboa-emacs debug-repl) + (jerboa-emacs async) (std net tcp)) (define pass-count 0) @@ -28,6 +30,32 @@ (flush-output-port (current-output-port)))))))) ;; ============================================================================ +;; Master timer for test context +;; ============================================================================ + +;; The debug REPL uses schedule-periodic! for non-blocking I/O polling. +;; In the Qt app, qt-app-exec! drives master-timer-tick!. In tests, we +;; need our own timer thread. +(define *test-timer-thread* #f) + +(define (start-test-timer!) + (set! *test-timer-thread* + (fork-thread + (lambda () + (let loop () + (sleep (make-time 'time-duration 50000000 0)) + (master-timer-tick!) + (loop)))))) + +(define (stop-test-timer!) + (when *test-timer-thread* + (guard (e [#t #f]) + (thread-terminate! *test-timer-thread*)) + (set! *test-timer-thread* #f))) + +(start-test-timer!) + +;; ============================================================================ ;; Helpers ;; ============================================================================ @@ -146,11 +174,7 @@ ;; ,threads (let ((resp (send-command in out ",threads"))) (check (> (length resp) 0) => #t) - (check (has-line? "debug-repl" resp) => #t)) - - ;; ,buffers - (let ((resp (send-command in out ",buffers"))) - (check (list? resp) => #t)) + (check (has-line? "master-timer" resp) => #t)) ;; ,help (let ((resp (send-command in out ",help"))) @@ -159,8 +183,8 @@ ;; ,state (let ((resp (send-command in out ",state"))) - (check (has-line? "buffers" resp) => #t) - (check (has-line? "threads" resp) => #t)) + (check (has-line? "listen-fd" resp) => #t) + (check (has-line? "bytes-allocated" resp) => #t)) ;; ,gc (let ((resp (send-command in out ",gc"))) @@ -207,24 +231,29 @@ (guard (e [#t #f]) (close-port out)))) ;; ============================================================================ -;; Test group 6: Multiple simultaneous clients +;; Test group 6: Sequential clients (single-client non-blocking design) ;; ============================================================================ -(display "--- debug-repl-multi-client ---\n") +(display "--- debug-repl-sequential-client ---\n") (flush-output-port (current-output-port)) +;; The thread-free REPL handles one client at a time. +;; Verify that after one client disconnects, another can connect. (with-fresh-server (port) - (let-values (((in1 out1) (connect-repl port)) - ((in2 out2) (connect-repl port))) - (sleep-ms 30) + ;; First client + (let-values (((in1 out1) (connect-repl port))) (skip-banner in1) + (let ((r1 (send-command in1 out1 "(+ 10 20)"))) + (check (has-line? "30" r1) => #t)) + (close-port in1) + (close-port out1)) + ;; Wait for server to notice disconnect + (sleep-ms 300) + ;; Second client + (let-values (((in2 out2) (connect-repl port))) (skip-banner in2) - (let ((r1 (send-command in1 out1 "(+ 10 20)")) - (r2 (send-command in2 out2 "(+ 30 40)"))) - (check (has-line? "30" r1) => #t) + (let ((r2 (send-command in2 out2 "(+ 30 40)"))) (check (has-line? "70" r2) => #t)) - (close-port in1) - (close-port out1) (close-port in2)