Strip jsh to compact shell
ober
b2ec3cddc9847ecb883514d17c46a51845a6947d
--- a/.build.yml +++ b/.build.yml @@ -5,25 +5,15 @@ packages: - uuid-dev - libz-dev - liblz4-dev - - libpcre2-dev - - libssl-dev - - libx11-dev - - pkg-config - git - curl - - unzip - - zip sources: - https://git.sr.ht/~lisp/jerboa - https://git.sr.ht/~lisp/jerboa-shell tasks: - build-jerboa-tools: | cd jerboa - # Build the multicall Jerboa tool artifact used by jerboa-shell. make jerboa - - build-shell: | - cd jerboa-shell - JERBUILD=$HOME/jerboa/dist/jerbuild make jsh-compile - test-shell: | cd jerboa-shell JERBUILD=$HOME/jerboa/dist/jerbuild make test deleted file mode 100644 --- a/.dockerignore +++ /dev/null @@ -1,40 +0,0 @@ -# Build artifacts — rebuilt inside container -**/*.so -**/*.wpo -**/*.o -**/*.a -*.boot -jsh -jsh-musl -jsh-musl.sha256 -jsh-macos -jsh-macos.sha256 -jsh-linux-amd64 -jsh-linux-amd64.sha256 -jsh-freebsd -jsh-freebsd.sha256 -jsh-android -jsh-android.sha256 -jsh_program.h -jsh_petite_boot.h -jsh_scheme_boot.h -jsh_jsh_boot.h - -# Staging dirs — local build output, rebuilt fresh inside the container. -# _jerbuild-stage holds macOS ar-merge output incl. a mode-000 __.SYMDEF that -# breaks Podman's context tar if not excluded. -_jerbuild-stage/ -coreutils-stage/ -ssl-stage/ -aws-stage/ -android-stage/ - -# Vendored deps — cloned fresh inside container from GitHub -vendor/ - -# VCS -.git - -# Docker -Dockerfile -.dockerignore --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,6 @@ on: branches: [master] pull_request: -permissions: - contents: write - jobs: test: runs-on: ubuntu-latest @@ -16,373 +13,4 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run unit tests - run: make test JERBOA=/build/mine/jerboa/lib SCHEME=scheme - - rust-musl: - runs-on: ubuntu-latest - container: - image: jerboa21/jerboa:latest - steps: - - uses: actions/checkout@v4 - - name: Cache Rust registry - uses: actions/cache@v4 - with: - path: ~/.cargo/registry - key: rust-musl-${{ hashFiles('patches/regex_native.rs', 'rust-coreutils/Cargo.lock') }} - restore-keys: rust-musl- - - name: Build Rust libs (x86_64-musl) - run: | - cd /build/mine/jerboa/jerboa-native-rs - cp $GITHUB_WORKSPACE/patches/regex_native.rs src/regex_native.rs - grep -q '#[cfg(feature = "duckdb")]' src/lib.rs || \ - sed -i 's/^mod duckdb_native;/#[cfg(feature = "duckdb")]\nmod duckdb_native;/' src/lib.rs - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate" \ - cargo build --release --target x86_64-unknown-linux-musl --no-default-features --features "tls crypto sqlite" - strip -S target/x86_64-unknown-linux-musl/release/libjerboa_native.a - cp target/x86_64-unknown-linux-musl/release/libjerboa_native.a $GITHUB_WORKSPACE/ - - cd $GITHUB_WORKSPACE/rust-coreutils - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate" \ - cargo build --release --target x86_64-unknown-linux-musl - strip -S target/x86_64-unknown-linux-musl/release/libjsh_coreutils.a - cp target/x86_64-unknown-linux-musl/release/libjsh_coreutils.a $GITHUB_WORKSPACE/ - - uses: actions/upload-artifact@v4 - with: - name: rust-libs-musl - path: | - libjerboa_native.a - libjsh_coreutils.a - retention-days: 7 - - rust-freebsd: - runs-on: ubuntu-latest - container: - image: jerboa21/jerboa:latest - steps: - - uses: actions/checkout@v4 - - name: Cache Rust registry - uses: actions/cache@v4 - with: - path: ~/.cargo/registry - key: rust-freebsd-${{ hashFiles('patches/regex_native.rs', 'rust-coreutils/Cargo.lock') }} - restore-keys: rust-freebsd- - - name: Install FreeBSD cross target - run: rustup target add x86_64-unknown-freebsd - - name: Cache FreeBSD sysroot - id: cache-freebsd-sysroot - uses: actions/cache@v4 - with: - path: /opt/freebsd-sysroot - key: freebsd-sysroot-14.4-v1 - - name: Install FreeBSD sysroot link libs - if: steps.cache-freebsd-sysroot.outputs.cache-hit != 'true' - run: | - # Some transitive deps (e.g. crc-fast via uucore) declare cdylib and - # therefore need the FreeBSD link libs (libexecinfo, libkvm, ...). - # Extract just the libs from the FreeBSD 14.4 base release tarball. - # 14.0 was removed from download.freebsd.org; 14.4 is the latest 14.x. - command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl xz-utils) - mkdir -p /opt/freebsd-sysroot - curl -fsSL --retry 3 -o /tmp/freebsd-base.txz \ - https://download.freebsd.org/releases/amd64/14.4-RELEASE/base.txz - tar -xJf /tmp/freebsd-base.txz -C /opt/freebsd-sysroot ./lib ./usr/lib - rm -f /tmp/freebsd-base.txz - - name: Build Rust libs (x86_64-freebsd) - run: | - # libc.so on FreeBSD is a linker script that references /lib/libc.so.7 - # and /usr/lib/libc_nonshared.a as ABSOLUTE paths. Plain `-L` flags - # don't redirect absolute paths — only `--sysroot` does. Without it, - # ld looks at the host's /lib/libc.so.7 (which doesn't exist on the - # Linux runner) and the link fails. - SYSROOT_FLAGS="-C link-arg=--sysroot=/opt/freebsd-sysroot" - LIB_FLAGS="-L /opt/freebsd-sysroot/usr/lib -L /opt/freebsd-sysroot/lib" - - cd /build/mine/jerboa/jerboa-native-rs - cp $GITHUB_WORKSPACE/patches/regex_native.rs src/regex_native.rs - grep -q '#[cfg(feature = "duckdb")]' src/lib.rs || \ - sed -i 's/^mod duckdb_native;/#[cfg(feature = "duckdb")]\nmod duckdb_native;/' src/lib.rs - # Drop cdylib for jerboa-native: only the static .a is needed by jsh-freebsd. - sed -i 's/^crate-type = \["cdylib", "staticlib"\]/crate-type = ["staticlib"]/' Cargo.toml - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate $SYSROOT_FLAGS $LIB_FLAGS" \ - cargo build --release --target x86_64-unknown-freebsd --no-default-features - strip -S target/x86_64-unknown-freebsd/release/libjerboa_native.a - cp target/x86_64-unknown-freebsd/release/libjerboa_native.a $GITHUB_WORKSPACE/ - - cd $GITHUB_WORKSPACE/rust-coreutils - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate $SYSROOT_FLAGS $LIB_FLAGS" \ - cargo build --release --target x86_64-unknown-freebsd - strip -S target/x86_64-unknown-freebsd/release/libjsh_coreutils.a - cp target/x86_64-unknown-freebsd/release/libjsh_coreutils.a $GITHUB_WORKSPACE/ - - uses: actions/upload-artifact@v4 - with: - name: rust-libs-freebsd - path: | - libjerboa_native.a - libjsh_coreutils.a - retention-days: 7 - - rust-android: - runs-on: ubuntu-latest - container: - image: jerboa21/jerboa:latest - steps: - - uses: actions/checkout@v4 - - name: Cache Rust registry - uses: actions/cache@v4 - with: - path: ~/.cargo/registry - key: rust-android-${{ hashFiles('patches/regex_native.rs', 'rust-coreutils/Cargo.lock') }} - restore-keys: rust-android- - - name: Install Android cross target - run: rustup target add aarch64-unknown-linux-musl - - name: Cache zig + cargo-zigbuild - id: cache-zig - uses: actions/cache@v4 - with: - path: | - /opt/zig - /usr/local/bin/cargo-zigbuild - key: zig-0.13.0-zigbuild-0.22.3 - - name: Install zig + cargo-zigbuild - if: steps.cache-zig.outputs.cache-hit != 'true' - shell: bash - run: | - # Use zig as the cross-linker for aarch64-linux-musl. Zig bundles its - # own musl libc/headers and downloads from ziglang.org / GitHub release - # mirrors which are far more reliable than musl.cc (which routinely - # times out from GitHub runners). cargo-zigbuild wraps cargo to use - # zig as the linker, transparently handling musl cross-link. - command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl xz-utils) - mkdir -p /opt/zig - ZIG_VERSION=0.13.0 - curl -fsSL --retry 5 --retry-delay 5 --max-time 300 \ - "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz" \ - -o /tmp/zig.tar.xz - tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 - rm -f /tmp/zig.tar.xz - # Pre-built cargo-zigbuild binary (avoid 5-min cargo install build) - ZIGBUILD_VERSION=0.22.3 - curl -fsSL --retry 5 --retry-delay 5 --max-time 300 \ - "https://github.com/rust-cross/cargo-zigbuild/releases/download/v${ZIGBUILD_VERSION}/cargo-zigbuild-x86_64-unknown-linux-musl.tar.xz" \ - -o /tmp/cargo-zigbuild.tar.xz - tar -xJf /tmp/cargo-zigbuild.tar.xz -C /usr/local/bin/ - rm -f /tmp/cargo-zigbuild.tar.xz - - name: Build Rust libs (aarch64-musl) - run: | - export PATH=/opt/zig:/usr/local/bin:$PATH - cd /build/mine/jerboa/jerboa-native-rs - cp $GITHUB_WORKSPACE/patches/regex_native.rs src/regex_native.rs - grep -q '#[cfg(feature = "duckdb")]' src/lib.rs || \ - sed -i 's/^mod duckdb_native;/#[cfg(feature = "duckdb")]\nmod duckdb_native;/' src/lib.rs - sed -i '/SYS_LANDLOCK_RESTRICT_SELF.*446;$/a\\n#[cfg(target_arch = "aarch64")]\nconst SYS_LANDLOCK_CREATE_RULESET: libc::c_long = 444;\n#[cfg(target_arch = "aarch64")]\nconst SYS_LANDLOCK_ADD_RULE: libc::c_long = 445;\n#[cfg(target_arch = "aarch64")]\nconst SYS_LANDLOCK_RESTRICT_SELF: libc::c_long = 446;' src/landlock.rs - # Drop cdylib for cross-compile: only the static .a is needed by jsh-android. - sed -i 's/^crate-type = \["cdylib", "staticlib"\]/crate-type = ["staticlib"]/' Cargo.toml - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate" \ - cargo zigbuild --release --target aarch64-unknown-linux-musl --no-default-features - strip -S target/aarch64-unknown-linux-musl/release/libjerboa_native.a - cp target/aarch64-unknown-linux-musl/release/libjerboa_native.a $GITHUB_WORKSPACE/ - - cd $GITHUB_WORKSPACE/rust-coreutils - RUSTFLAGS="--remap-path-prefix $HOME/.cargo/registry/src=crate" \ - cargo zigbuild --release --target aarch64-unknown-linux-musl - strip -S target/aarch64-unknown-linux-musl/release/libjsh_coreutils.a - cp target/aarch64-unknown-linux-musl/release/libjsh_coreutils.a $GITHUB_WORKSPACE/ - - uses: actions/upload-artifact@v4 - with: - name: rust-libs-android - path: | - libjerboa_native.a - libjsh_coreutils.a - retention-days: 7 - - build: - needs: rust-musl - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: rust-libs-musl - path: prebuilt/ - - name: Install Podman - run: | - sudo apt-get update - sudo apt-get install -y podman - podman --version - - name: Build jsh-musl (Podman, pre-built Rust libs injected) - run: | - make podman \ - JSH_PREBUILT_NATIVE=prebuilt/libjerboa_native.a \ - JSH_PREBUILT_COREUTILS=prebuilt/libjsh_coreutils.a - - name: Verify binary - run: | - ls -lh jsh-musl - file jsh-musl - cat jsh-musl.sha256 - - uses: actions/upload-artifact@v4 - with: - name: jsh-musl - path: | - jsh-musl - jsh-musl.sha256 - if-no-files-found: error - retention-days: 30 - - build-freebsd: - needs: rust-freebsd - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: rust-libs-freebsd - path: prebuilt/ - - name: Install Podman - run: | - sudo apt-get update - sudo apt-get install -y podman - podman --version - - name: Build jsh-freebsd (Podman, pre-built Rust libs injected) - run: | - make freebsd-podman \ - JSH_PREBUILT_NATIVE=prebuilt/libjerboa_native.a \ - JSH_PREBUILT_COREUTILS=prebuilt/libjsh_coreutils.a - - name: Verify binary - run: | - ls -lh jsh-freebsd - file jsh-freebsd - cat jsh-freebsd.sha256 - - uses: actions/upload-artifact@v4 - with: - name: jsh-freebsd - path: | - jsh-freebsd - jsh-freebsd.sha256 - if-no-files-found: error - retention-days: 30 - - build-android: - needs: rust-android - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Podman and QEMU - run: | - sudo apt-get update - sudo apt-get install -y podman qemu-user-static binfmt-support - sudo update-binfmts --enable qemu-aarch64 || true - podman --version - - uses: actions/download-artifact@v4 - with: - name: rust-libs-android - path: prebuilt/ - - name: Build jsh-android (Podman, pre-built Rust libs injected) - run: | - make android-podman \ - JSH_PREBUILT_NATIVE=prebuilt/libjerboa_native.a \ - JSH_PREBUILT_COREUTILS=prebuilt/libjsh_coreutils.a - - name: Verify binary - run: | - ls -lh jsh-android - file jsh-android - cat jsh-android.sha256 - - uses: actions/upload-artifact@v4 - with: - name: jsh-android - path: | - jsh-android - jsh-android.sha256 - if-no-files-found: error - retention-days: 30 - - build-macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - name: Install dependencies - run: | - brew install openssl@3 lz4 - echo "PKG_CONFIG_PATH=$(brew --prefix openssl@3)/lib/pkgconfig" >> $GITHUB_ENV - - name: Resolve Chez Scheme HEAD SHA - id: chez-sha - run: | - SHA=$(git ls-remote https://github.com/ober/ChezScheme.git refs/heads/main | cut -f1) - echo "sha=$SHA" >> "$GITHUB_OUTPUT" - - name: Cache our Chez Scheme install - id: cache-chez - uses: actions/cache@v4 - with: - path: | - /opt/homebrew/lib/csv10.4.0-pre-release.4 - /opt/homebrew/bin/scheme - /opt/homebrew/bin/petite - /opt/homebrew/bin/scheme-script - key: chez-macos-arm64-${{ steps.chez-sha.outputs.sha }} - - name: Build our Chez Scheme fork - if: steps.cache-chez.outputs.cache-hit != 'true' - run: | - # jerboa-shell needs ober/ChezScheme (10.4.0-pre-release.4+) which adds - # base64-encode/decode, sha256-bytevector, etc. Stock Cisco Chez 10.3 - # and Homebrew chezscheme are too old. - git clone --depth 1 --branch main https://github.com/ober/ChezScheme.git /tmp/ChezScheme - cd /tmp/ChezScheme - ./configure --installprefix=/opt/homebrew - make -j$(sysctl -n hw.ncpu) - sudo make install - - name: Verify Chez Scheme - run: | - which scheme - scheme --version - echo '(display (top-level-bound? (quote base64-encode))) (newline)' | scheme --script /dev/stdin - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - name: Cache Rust registry - uses: actions/cache@v4 - with: - path: ~/.cargo/registry - key: rust-macos-${{ hashFiles('patches/regex_native.rs', 'rust-coreutils/Cargo.lock') }} - restore-keys: rust-macos- - - name: Build jsh-macos - run: make jsh-macos - - name: Smoke-test binary - run: | - ls -lh jsh-macos - ./jsh-macos -c 'echo ok' - - uses: actions/upload-artifact@v4 - with: - name: jsh-macos - path: | - jsh-macos - jsh-macos.sha256 - if-no-files-found: error - retention-days: 30 - - release: - needs: [test, build, build-freebsd, build-android, build-macos] - if: github.ref == 'refs/heads/master' && github.event_name == 'push' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - path: artifacts - merge-multiple: true - - name: Update rolling 'latest' release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - TAG=latest - gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create "$TAG" \ - --repo "${{ github.repository }}" \ - --title "Latest static jsh binaries" \ - --notes "Auto-built by CI from latest master commit." \ - --target master - gh release upload "$TAG" \ - artifacts/jsh-musl artifacts/jsh-musl.sha256 \ - artifacts/jsh-freebsd artifacts/jsh-freebsd.sha256 \ - artifacts/jsh-android artifacts/jsh-android.sha256 \ - artifacts/jsh-macos artifacts/jsh-macos.sha256 \ - --repo "${{ github.repository }}" --clobber + run: make test --- a/.gitignore +++ b/.gitignore @@ -4,104 +4,28 @@ *.wpo *.o *.boot -.jerbuild-hashes +*.dylib *.tarm64osx +*.tx86_64osx +*.ta6le +*.ta6fb + /jsh -/jsh-musl -/jsh-musl.sha256 -/jsh-android -/jsh-android.sha256 /jsh-macos /jsh-macos.sha256 -/jsh-macos-base -/jsh-macos-base.sha256 -/jsh-macos-full -/jsh-macos-full.sha256 -/jsh-freebsd -/jsh-freebsd.sha256 -/jsh-kernel -/jsh-linux-amd64 -/jsh-linux-amd64-main.c -/jsh-linux-arm64 -/jsh-linux-arm64-main.c -/jsh-freebsd-amd64 -/jsh-freebsd-amd64-main.c -/petite_boot.h -/scheme_boot.h -/jsh_libs_boot.h -/jsh_program.h -/qemu_*.core -gsh_program.h -jsh-all.so -libgsh-ffi.so - -# Auto-generated .sls files (built by build-jerboa.ss from .ss sources) -# Handwritten files are explicitly tracked via git add -f -src/jsh/arithmetic.sls -src/jsh/ast.sls -src/jsh/builtins.sls -src/jsh/completion.sls -src/jsh/control.sls -src/jsh/environment.sls -src/jsh/executor.sls -src/jsh/expander.sls -src/jsh/functions.sls -src/jsh/fuzzy.sls -src/jsh/fzf.sls -src/jsh/glob.sls -src/jsh/history.sls -src/jsh/jobs.sls -src/jsh/lexer.sls -src/jsh/lineedit.sls -src/jsh/macros.sls -src/jsh/main.sls -src/jsh/parser.sls -src/jsh/pipeline.sls -src/jsh/pregexp-compat.sls -src/jsh/prompt.sls -src/jsh/procwatch.sls -src/jsh/redirect.sls -src/jsh/registry.sls -src/jsh/script.sls -src/jsh/signals.sls -src/jsh/startup.sls -src/jsh/util.sls -src/jsh/player.sls -src/jsh/recorder.sls -src/jsh/embed-data.sls - -# Embedded files (may contain secrets) -embed/ - -# Vendor -_vendor/ -vendor/ -/jerboa-shell/ - -# Rust build artifacts -rust-coreutils/target/ -rust-coreutils/ripgrep-core/target/ +/ffi-shim-symbols.list -# Android build staging -/android-stage/ -/aws-stage/ - -# macOS build vault staging (re-populated each build from upstream chez-fuse) -/vault-stage-macos/ - -# Legacy SSH build staging -/ssh-stage/ - -# Cache -.gerbil-lsp-cache/ -.jerboa/bin/ -jsh-freebsd -# jerbuild staging directory +# Generated source/staging /jsh-src/ /_jerbuild-stage/ -# generated at build time from jsh-generate.ss -/jsh-generated.ss -# Claude Code session files (user-specific) +/src/ +/lib/ +**/.jerbuild-hashes + +# Vendored external checkouts and local caches +/_vendor/ +/vendor/ +/.jerboa/bin/ +/.jcode/ /.claude/ -*.dylib -.jcode/ +.gerbil-lsp-cache/ deleted file mode 100644 --- a/.jerboa/security.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "version": 1, - "repo": "jerboa-shell", - "extends": ["jerboa:cli", "jerboa:ffi", "jerboa:parser", "jerboa:generated-heavy"], - "paths": { - "production": ["*.ss", "*.sls", "lib/**/*.ss", "lib/**/*.sls", "src/**/*.{ss,sls,c,h,rs}", "support/**", "ffi-shim.c", "Makefile"], - "tests": ["test/**", "tests/**", "**/*-test.ss", "fixtures/**"], - "generated": ["build/**", "dist/**", "target/**", "_tmp/**", "jsh-generated.ss", "jsh-macos", "*.sha256", "*.so", "*.dylib", "*.wpo"], - "vendor": ["vendor/**", "third_party/**", "oils/**"], - "docs": ["README.md", "docs/**", "*.md", "AGENTS.md"] - }, - "policy": { - "failOn": ["critical", "high"], - "imports": { "directChezscheme": "allow-in-ffi-boundaries" }, - "ffi": { "allowed": true, "requireCloseOnExec": true, "requireDynamicWindCleanup": true }, - "process": { "shellInterpolation": "deny", "restrictedModeExpected": true }, - "network": { "requireTimeouts": true }, - "eval": { "stringEval": "deny", "bareRead": "deny", "allowReadEval": false }, - "shell": { "treatScriptsAsCodeExecution": true, "requireFdHygiene": true } - }, - "suppressions": [] -} --- a/.jerbuild +++ b/.jerbuild @@ -1,57 +1,13 @@ -;; jerbuild build config for jerboa-shell (jsh) — macOS. -;; -;; All paths are relative to this file. The pre-build hook regenerates -;; jsh-generated.ss + stages stdlib overrides into _jerbuild-stage. -;; -;; Out: jsh-macos (matches what `make jsh-macos` produces). Run with -;; jerbuild build -;; from this directory. +;; jerbuild build config for jsh on macOS. -(entry "jsh-generated.ss") +(entry "jsh.ss") (output "jsh-macos") -;; Requires: jerbuild, cargo/rustc, cc, and libssl/libcrypto. -;; The pre-build hook uses jerbuild's bundled Chez/std library, sparse-cloned -;; vendor/jerboa-native-rs, and cargo-built Rust archives. No Jerboa source -;; checkout is required. -(libdirs "_jerbuild-stage" ; stdlib patches — must shadow bundle copies - "src" - "vendor/jerboa-coreutils/lib" - "vendor/jerboa-awk/lib" - "vendor/jerboa-sed/lib" - "vendor/jerboa-aws/lib" - "vendor/jerboa-wormhole" - "vendor/jerboa-yubikey/lib" - "vendor/jerboa-ssh/lib" - "vendor/jerboa-fuse/lib") +(libdirs "src") -;; FFI shim — implements all ffi_* foreign-procedure targets used by -;; (jsh ffi), (jsh limits), (std os landlock), etc. (extra-sources "ffi-shim.c") +(extra-ldflags "-lutil") -;; Rust archive staged by support/stage-for-jerbuild.sh. It builds -;; libjerboa_native.a + libjsh_coreutils.a with Cargo, then merges them into -;; a single archive with duplicate Rust runtime symbols localized. -(extra-archives - "_jerbuild-stage/librust_combined.a" - ;; jerboa-ssh agent + transport C shims, staged by stage-for-jerbuild.sh. - "_jerbuild-stage/libjerboa_ssh.a" - ;; jerboa-fuse mount C shim (mount_helper.c), staged by stage-for-jerbuild.sh. - "_jerbuild-stage/libjerboa_fuse.a") - -;; jerboa-ssh + jerboa-crypto need libssl/libcrypto from Homebrew. libutil is -;; needed by openpty/forkpty used in (jsh pty). -(extra-ldflags "-L/opt/homebrew/lib" "-L/usr/local/lib" - "-lssl" "-lcrypto" "-lutil") - -;; Custom main.c — jsh bypasses Chez's arg parsing (JSH_ARG* env vars) -;; and uses Sscheme_script instead of Sscheme_program to avoid the -;; fork-thread / GC futex deadlock. (main-c "support/jsh-jerbuild-main.c") - -;; FFI symbols registered via Sforeign_symbol after Sbuild_heap — static -;; macOS binaries can't resolve via dlsym(RTLD_DEFAULT). -(ffi-symbols "support/jsh-jerbuild-symbols.list") - -;; Pre-build: wipe stale .so/.wpo + patch stdlib + regen jsh-generated.ss. +(ffi-symbols "ffi-shim-symbols.list") (pre-build "support/stage-for-jerbuild.sh") deleted file mode 100644 --- a/.jerbuild.freebsd-amd64 +++ /dev/null @@ -1,46 +0,0 @@ -;; jerbuild build config for jerboa-shell (jsh) — native FreeBSD amd64. -;; -;; Build on a FreeBSD amd64 host with: -;; jerbuild build --config .jerbuild.freebsd-amd64 -;; -;; This uses only the installed jerbuild bundle, cc, and cargo. It deliberately -;; has no source-checkout, cross-compiler, or absolute runtime-path dependency. - -(entry "jsh-generated.ss") -(output "jsh-freebsd") - -(cc "cc") - -;; FreeBSD libc has dlopen, no separate -ldl. -lutil is for openpty. -;; Chez's expeditor needs termcap, and the packaged Chez kernel references -;; GNU libiconv from /usr/local/lib. -(os-libs "-lm -lpthread -lutil -ltermcap -L/usr/local/lib -liconv") - -(libdirs "_jerbuild-stage" ; stdlib patches — must shadow bundle copies - "src" - "vendor/jerboa-coreutils/lib" - "vendor/jerboa-awk/lib" - "vendor/jerboa-sed/lib" - "vendor/jerboa-aws/lib" - "../jerboa-wormhole" - "vendor/jerboa-yubikey/lib" - "vendor/jerboa-ssh/lib" - "vendor/jerboa-fuse/lib") - -;; FFI shim — same shim used by the macOS + Linux builds. Embed crypto symbols -;; come from the staged jerboa-native-rs archive. -(extra-sources "ffi-shim.c") - -;; Rust archive staged by support/stage-for-jerbuild-freebsd.sh. No coreutils -;; archive is linked on FreeBSD yet; freebsd-main.c provides weak stubs. -(extra-archives - "_jerbuild-stage/libjerboa_native.a" - "_jerbuild-stage/libjerboa_ssh.a" - "_jerbuild-stage/libjerboa_fuse.a") - -(extra-ldflags "-Wl,--export-dynamic") - -;; FreeBSD-specific main.c generated by the pre-build hook. -(main-c "_jerbuild-stage/freebsd-main.c") - -(pre-build "support/stage-for-jerbuild-freebsd.sh") deleted file mode 100644 --- a/.jerbuild.linux-amd64 +++ /dev/null @@ -1,47 +0,0 @@ -;; jerbuild build config for jerboa-shell (jsh) — native Linux amd64. -;; -;; Build on a Linux amd64 host with: -;; jerbuild build --config .jerbuild.linux-amd64 -;; -;; This intentionally uses only the installed jerbuild bundle, cc, and cargo. -;; There is no Jerboa source checkout, cross compiler, or absolute runtime path. - -(entry "jsh-generated.ss") -(output "jsh-linux-amd64") - -(cc "cc") - -(os-libs "-lm -ldl -lpthread -ltinfo") - -(libdirs "_jerbuild-stage" ; stdlib patches — must shadow bundle copies - "src" - "vendor/jerboa-coreutils/lib" - "vendor/jerboa-awk/lib" - "vendor/jerboa-sed/lib" - "vendor/jerboa-aws/lib" - "../jerboa-wormhole" - "vendor/jerboa-yubikey/lib" - "vendor/jerboa-ssh/lib" - "vendor/jerboa-fuse/lib") - -;; FFI shim — same shim used by the macOS build. Embed crypto symbols come -;; from the staged jerboa-native-rs archive. -(extra-sources "ffi-shim.c") - -;; Rust archives staged by support/stage-for-jerbuild-linux.sh. -(extra-archives - "_jerbuild-stage/libjerboa_native.a" - "_jerbuild-stage/libjsh_coreutils.a" - "_jerbuild-stage/libjerboa_ssh.a" - "_jerbuild-stage/libjerboa_fuse.a") - -;; --export-dynamic + Sforeign_symbol registrations in main.c are how Chez -;; foreign-procedure finds symbols. --allow-multiple-definition tolerates -;; duplicate Rust runtime symbols across the two static archives. -(extra-ldflags "-Wl,--export-dynamic" - "-Wl,--allow-multiple-definition") - -;; Linux-specific main.c is generated by the pre-build hook. -(main-c "_jerbuild-stage/linux-main.c") - -(pre-build "support/stage-for-jerbuild-linux.sh") deleted file mode 100644 --- a/.jerbuild.linux-amd64-native +++ /dev/null @@ -1,35 +0,0 @@ -;; Compatibility alias for older local commands. The canonical native Linux -;; config is .jerbuild.linux-amd64. - -(entry "jsh-generated.ss") -(output "jsh-linux-amd64") - -(cc "cc") - -(os-libs "-lm -ldl -lpthread -ltinfo") - -(libdirs "_jerbuild-stage" - "src" - "vendor/jerboa-coreutils/lib" - "vendor/jerboa-awk/lib" - "vendor/jerboa-sed/lib" - "vendor/jerboa-aws/lib" - "../jerboa-wormhole" - "vendor/jerboa-yubikey/lib" - "vendor/jerboa-ssh/lib" - "vendor/jerboa-fuse/lib") - -(extra-sources "ffi-shim.c") - -(extra-archives - "_jerbuild-stage/libjerboa_native.a" - "_jerbuild-stage/libjsh_coreutils.a" - "_jerbuild-stage/libjerboa_ssh.a" - "_jerbuild-stage/libjerboa_fuse.a") - -(extra-ldflags "-Wl,--export-dynamic" - "-Wl,--allow-multiple-definition") - -(main-c "_jerbuild-stage/linux-main.c") - -(pre-build "support/stage-for-jerbuild-linux.sh") --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,263 +1,23 @@ -# jerboa-shell +# jerboa-shell Notes -## Chez Scheme Dependency — OUR Chez, Not THE Chez +This repository is the compact `jsh` shell. Keep it focused on bash/zsh-style +shell behavior: parsing, expansion, builtins, redirection, pipelines, jobs, +history, completion, process execution, and compatibility/performance tests. -**This project depends on the in-tree ChezScheme at `~/mine/ChezScheme`, NOT the upstream Cisco release or any package-manager build (Homebrew, apt, pkg, pacman, etc.).** +## Build -`~/mine/ChezScheme` is a fork that ships ahead of upstream — it adds new core builtins (`base64-encode`, `base64-decode`, `bytevector-append`, `sha1-bytevector`, `sha256-bytevector`, …) that jerboa-shell, jerboa, and the vendored libraries rely on as plain `(import (chezscheme))` symbols. Stock Cisco Chez (10.3 and below) does not have these and will fail with "attempt to reference unbound identifier" or, after we patch around it, "multiple definitions" when our overrides collide. - -### macOS setup (one-time) - -If `scheme --version` prints `10.3.0` or anything before `10.4.0-pre-release.4`, you have the wrong Chez. Replace it: - -```sh -brew uninstall chezscheme # remove Homebrew's stock build -cd ~/mine/ChezScheme -./configure --installprefix=/opt/homebrew # writable on Apple Silicon, no sudo -make -make install -scheme --version # → 10.4.0-pre-release.4 (or newer) -``` - -Verify the new builtins are present: ```sh -echo '(display (top-level-bound? (quote base64-encode))) (newline)' | scheme --script /dev/stdin -# → #t +make jsh-macos +./jsh-macos -c 'echo ok' ``` -If it prints `#f`, the wrong scheme is in `PATH` — `which scheme` should be `/opt/homebrew/bin/scheme` (a symlink into `/opt/homebrew/lib/csv10.4.0-pre-release.4/tarm64osx/scheme`). - -### Linux / FreeBSD / Android - -There is no Podman/container build. `make linux-amd64`/`linux-arm64` cross-build -from any host using the cross Chez under `vendor/jerboa/.chez` (built by the -`chez-cross-*` Makefile targets) — no host Chez install needed for the cross -path. `make binary` on a Linux/FreeBSD/Android host uses that host's Chez -(our 10.4; the `jerboa21/jerboa` image and the dns2 FreeBSD box already have it). - -### Why we don't keep a fallback - -Earlier versions of `lib/std/text/base64.sls` shipped a pure-Scheme port so the build worked on stock Chez 10.3. That fallback is **gone** — keeping two parallel implementations was a maintenance burden, and stock Chez was always going to fall further behind as more builtins land. A single source of truth (our Chez, with the new prims) is the long-term answer. - -If you find yourself wanting to "make it work on stock Chez" — don't. Build our Chez instead. - -## NEVER Remove Features — ALWAYS Fix - -**Never stub out, comment out, delete, or disable existing functionality** to work around a build error, missing symbol, or platform gap. If something is broken, **fix the root cause**. Removing features is never an acceptable solution. - -This applies especially to: -- Platform-specific code (FreeBSD, Linux, Android) — a missing symbol or build error on one platform must be fixed, not stubbed with a no-op or `(void)` -- FFI bindings — if a symbol is missing, add it; don't remove the call -- Commands and their subcommands — if a feature breaks, restore it; don't silently drop it - -When you encounter a missing symbol, build failure, or runtime error: **investigate and fix it**. If you cannot fix it immediately, say so — do not quietly remove the feature. - -## As on Linux, On FreeBSD and macOS - -Every feature, build step, and capability must work on Linux, FreeBSD, and macOS. If you add something to one platform's build, add the equivalent to the others. No platform-specific gaps. No stubs. No "FreeBSD/macOS later." - -## Build Scripts — MANDATORY: Apply Changes to ALL Platforms - -There are **four** platform build scripts. Any fix or change to one MUST be mirrored to all others (adjusted for each platform's dep layout). Do NOT fix Android and stop. +## Verification -- `build-jsh-android.sh` / `build-jsh-android.ss` — Termux/Android (uses `vendor/jerboa/lib` as staged tree) -- `build-jsh-musl.sh` / `build-jsh-musl.ss` — static Linux via musl -- `build-jsh-freebsd.sh` / `build-jsh-freebsd.ss` — static FreeBSD -- `build-jsh-macos.sh` / `build-jsh-macos.ss` — macOS - -Examples of changes that MUST be applied to all four: -- Removing a dependency (e.g. dropping `LEGACY_RUNTIME_DIR` from env, exports, echoes, and `LIBDIRS`) -- Adding a local stdlib shim (e.g. copying `lib/std/misc/cpu.sls` into the jerboa lib dir when upstream is missing it) -- Adding/renaming FFI symbol whitelist entries (see next section) -- New Scheme staging steps, new dep dirs, new Rust static libs to link - -**Verification after changing build scripts:** ```sh -grep -in "<symbol>" build-jsh-*.sh build-jsh-*.ss -``` -should show a consistent picture across platforms. If one script has a rule and another doesn't, the change is incomplete. - -The destination paths differ per platform — Android copies into `${VENDOR}/jerboa/lib/...` while musl/freebsd/macos copy into `${JERBOA_DIR}/...` (JERBOA_DIR already points at the lib dir). The *intent* is identical; the paths differ. - -## FFI Symbol Whitelist — AUTO-GENERATED from ffi-shim.c - -The symbol whitelist that controls which C functions are exported to Scheme is -**`ffi-shim-symbols.list`, auto-generated from `ffi-shim.c`** by -`tools/extract-ffi-symbols.sh` (a Makefile prerequisite, regenerated whenever -`ffi-shim.c` changes). `build-jsh-cross.ss` reads it to emit the externs + -registration. - -**So a new non-static function in `ffi-shim.c` is picked up automatically** — no -manual per-platform whitelist edit needed. (Older CLAUDE.md guidance to hand-edit -an `ffi-shim-symbols` list in each `build-jsh-*.ss` is obsolete.) After adding a -function, you can confirm with `tools/extract-ffi-symbols.sh --ffi-shim | grep <name>`. - -**Failure to do this will cause runtime crashes on the platforms you missed.** This is the #1 cause of "works on Linux, crashes on FreeBSD" bugs. Search for a nearby symbol (e.g. `ffi_stream_listen_tcp`) to find the right location in each file. - -## make vs gmake — ALWAYS Use gmake on FreeBSD - -**Always invoke `gmake` (GNU make), never `make` (bmake).** The Makefile uses GNU make syntax (`ifeq`, `else ifeq`, `$(shell ...)`, etc.) which is incompatible with BSD make. On FreeBSD, `/usr/bin/make` is bmake and will fail with parse errors. - -- Use: `gmake jsh-freebsd`, `gmake test`, `gmake clean`, etc. -- Never use: `make jsh-freebsd` (bmake — will error) -- **Never modify the Makefile to accommodate bmake.** The Makefile is GNU make and must stay that way. - -## Build — MANDATORY: Always Full Rebuild - -After modifying ANY source file (`.sls`, `.ss`, `.c`, or build scripts), ALWAYS -rebuild before testing. **There is NO Podman/Docker build and NO `jsh-musl` -target — never use or recommend them.** - -The only build commands: - -- **`make binary`** — native build for the current host. THE everyday build. - macOS → `jsh-macos`; Linux → static-musl `jsh-linux-<arch>` (glibc - `jsh-linux-native` only if `musl-gcc` is absent); FreeBSD → `jsh-freebsd-<arch>`. -- **`make linux-amd64`** / **`make linux-arm64`** — cross-build a fully static - Linux ELF from ANY host via `build-jsh-cross.ss` (Chez xpatch + musl-cross, - NO Podman/QEMU). `scp` the result to a Linux box (e.g. linux.cons.io) to run. -- **`make freebsd-amd64`** / **`make android`** — native-only: build on a FreeBSD - amd64 host (e.g. dns2.linbsd.org) / a Termux device. No off-platform cross path. - -**NEVER test with a binary you didn't just build.** Rebuild FIRST. - -### Wiring build dependencies — touch the scripts the real builds use - -When a feature needs new libdirs / archives / FFI symbols / vendored libs, wire it -where `make binary` and `make linux-amd64` actually read from — NOT only the -`.jerbuild` jerbuild configs: -- native: `build-jsh.ss` / `build-binary-jsh.ss` (and the per-os `build-jsh-<os>.ss`) -- cross: `build-jsh-cross.ss` and `XC_LIBDIRS` in the Makefile -- `ffi-shim-symbols.list` is auto-generated from `ffi-shim.c` - (`tools/extract-ffi-symbols.sh`); new `ffi-shim.c` functions are picked up - automatically — no manual per-platform whitelist edit needed. - -Verify on macOS with `make binary` (≡ `jsh-macos`); verify Linux with -`make linux-amd64` + scp to a Linux box. macOS has no Podman concerns because -no build uses Podman. - -### Stale Artifact Prevention — ALREADY HANDLED BY MAKEFILE - -The `jerboa` Makefile target automatically deletes all `.so` and `.wpo` caches under `src/jsh/` before regenerating `.sls` files. This prevents the #1 class of "my edit had no effect" bugs: Chez Scheme reusing a cached `.so` that was compiled from an older `.sls`. - -**If you suspect stale artifacts despite this** (e.g. you edited `.sls` directly, which you shouldn't): -```bash -find src -name "*.so" -delete && find src -name "*.wpo" -delete -``` - -**Root cause**: The build chain is `.ss` → `.sls` (build-jerboa.ss) → `.so` (Chez compiler). Chez's `--compile-imported-libraries` skips recompilation if the `.so` is newer than the `.sls`. If the `.sls` regeneration is skipped (timestamps) but you changed something upstream, the stale `.so` gets baked into the binary. The Makefile now prevents this by always cleaning caches before the `.ss` → `.sls` step. - -### Post-Build Smoke Test — MANDATORY after every build - -Only test binaries that can actually execute on the current platform: - -- **macOS**: test `jsh-macos` only — never try to run `jsh-linux-*`, `jsh-freebsd-*`, or `jsh-android` -- **Linux**: test `jsh-linux-<arch>` only -- **FreeBSD**: test `jsh-freebsd-<arch>` only -- **Android/Termux**: test `jsh-android` only - -```bash -./jsh-macos -c 'echo ok' # on macOS -./jsh-linux-amd64 -c 'echo ok' # on Linux (or run it on the box you scp'd to) -./jsh-freebsd-amd64 -c 'echo ok' # on FreeBSD -./jsh-android -c 'echo ok' # on Android +make test +make test-binary +make compat-smoke ``` -`make test-binary` already skips binaries that can't execute on the current platform. - -If this fails (e.g. missing FFI symbol, crash on startup), **the build is broken**. Fix it before telling the user it's ready. A binary that compiles but doesn't run is not a successful build. - -### Rust Native Library - -All four build scripts (android, musl, freebsd, macos) resolve -`libjerboa_native.a` from `vendor/jerboa/jerboa-native-rs/...`, with explicit -environment-variable overrides for manual builds. - -If you modify Rust code, rebuild it in whichever directory the build script resolved. `make vendor-deps` refreshes the vendored copy from upstream. - -### Verification — BLOCKING requirement before commit/push - -You MUST NOT commit or push until ALL of the following pass. No exceptions, no "pre-existing issue" excuses. - -1. `make binary` (native) and/or `make linux-amd64` (cross) completes with exit code 0 -2. The freshly built binary runs and prints `ok` (e.g. `./jsh-macos -c 'echo ok'`, - or scp `jsh-linux-amd64` to a Linux box and run it there) -3. `make test-binary` — ALL functional tests pass (0 failures) -