Harden process isolation and FFI loading
ober
52695eeffb1e08cf8998117436d28753288231da
--- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: test-audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install system tools run: | --- a/.github/workflows/security-baseline.yml +++ b/.github/workflows/security-baseline.yml @@ -13,7 +13,7 @@ jobs: baseline: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Required release files run: | --- a/.jerbuild +++ b/.jerbuild @@ -5,7 +5,8 @@ (libdirs "src") -(extra-sources "ffi-shim.c") +(extra-sources "ffi-shim.c" + "support/jsh-program-image.c") (extra-ldflags "-lutil") (main-c "support/jsh-jerbuild-main.c") --- a/Makefile +++ b/Makefile @@ -36,16 +36,31 @@ CC ?= cc WARN_CFLAGS ?= -Wall -Wextra -Wformat=2 -Wshadow -Wpointer-arith -Wcast-align -Wwrite-strings HARDEN_CFLAGS ?= -O2 -fPIC -fstack-protector-strong -D_FORTIFY_SOURCE=2 FFI_CFLAGS ?= $(HARDEN_CFLAGS) $(WARN_CFLAGS) +JSH_FFI_ENV = JSH_FFI_DEV_NATIVE=1 JSH_FFI_LIB="$(CURDIR)" +NATIVE_TEST_DIR ?= $(CURDIR)/dist/native-tests +NATIVE_TEST_CFLAGS ?= -O1 -g -Wall -Wextra -Werror -Wformat=2 -Wshadow -Wconversion -Wsign-conversion +SANITIZER_CC ?= $(CC) +ifeq ($(UNAME_S),Darwin) + # Apple ASan can deadlock around the fork+pthread wait-status stress case. + # UBSan covers this host; Linux CI/runners use the combined ASan+UBSan gate. + SANITIZER_CFLAGS ?= -fsanitize=undefined -fno-omit-frame-pointer +else + SANITIZER_CFLAGS ?= -fsanitize=address,undefined -fno-omit-frame-pointer +endif SHELL_MODULES = ast registry macros pregexp-compat util environment lexer arithmetic glob fuzzy \ history parser functions signals expander redirect control jobs builtins \ pipeline executor completion prompt lineedit script startup main OILS_DIR := _vendor/oils +OILS_REPOSITORY ?= https://github.com/oils-for-unix/oils.git +OILS_COMMIT ?= 15de8fd779569e6e3a9f5fcbfc00e7df0ebe0380 +OILS_TREE ?= f95f91004abcfdb82dd413e6ef2a7c71591f23d6 SH_SPEC := python3 $(CURDIR)/test/run_spec.py BASH := /bin/bash JSH ?= $(if $(filter Darwin,$(UNAME_S)),./jsh-macos,./jsh) -.PHONY: ensure-jerboa-tools jerboa binary jsh jsh-macos macos run security test test-binary adversarial-corpus audit \ +.PHONY: ensure-jerboa-tools jerboa binary jsh jsh-macos macos run security test test-binary test-native \ + test-native-sanitize test-security-regressions adversarial-corpus audit \ sbom reproducibility-report timing-evidence verify release-evidence \ compat compat-smoke compat-tier0 compat-tier1 compat-tier2 compat-one compat-range compat-debug \ bench clean help @@ -59,9 +74,9 @@ ensure-jerboa-tools: sh support/ensure-jerboa.sh "$(JERBOA_VERSION)" "$(JERBOA_TOOL_DIR)"; \ fi -ffi-shim-symbols.list: ffi-shim.c tools/extract-ffi-symbols.sh +ffi-shim-symbols.list: ffi-shim.c tools/extract-ffi-symbols.sh Makefile @echo "=== Regenerating FFI symbol whitelist from ffi-shim.c ===" - tools/extract-ffi-symbols.sh --ffi-shim > $@.tmp && mv $@.tmp $@ + @tools/extract-ffi-symbols.sh --ffi-shim > $@.tmp && mv $@.tmp $@ $(FFI_LIB): ffi-shim.c @echo "=== Building interpreted FFI shim: $@ ===" @@ -90,18 +105,46 @@ jsh: binary @ls -lh jsh run: ensure-jerboa-tools $(FFI_LIB) jerboa - $(JERBUILD) exec --libdirs "$(LIBDIRS)" jsh.ss + $(JSH_FFI_ENV) $(JERBUILD) exec --libdirs "$(LIBDIRS)" jsh.ss security: @REPO_ROOT="$(CURDIR)" sh tools/security-check.sh test: ensure-jerboa-tools $(FFI_LIB) jerboa @echo "=== Running unit tests ===" - $(JERBUILD) exec --libdirs "$(LIBDIRS)" test/test-jsh.ss + $(JSH_FFI_ENV) $(JERBUILD) exec --libdirs "$(LIBDIRS)" test/test-jsh.ss + +test-native: + @mkdir -p "$(NATIVE_TEST_DIR)" + $(CC) -std=c11 -DJSH_FFI_TESTING $(NATIVE_TEST_CFLAGS) \ + ffi-shim.c test/test-ffi-concurrency.c -pthread \ + -o "$(NATIVE_TEST_DIR)/test-ffi-concurrency" + $(CC) -std=c11 $(NATIVE_TEST_CFLAGS) \ + support/jsh-program-image.c test/test-program-image.c \ + -o "$(NATIVE_TEST_DIR)/test-program-image" + "$(NATIVE_TEST_DIR)/test-ffi-concurrency" + "$(NATIVE_TEST_DIR)/test-program-image" + +test-native-sanitize: + @mkdir -p "$(NATIVE_TEST_DIR)" + $(SANITIZER_CC) -std=c11 -DJSH_FFI_TESTING $(NATIVE_TEST_CFLAGS) $(SANITIZER_CFLAGS) \ + ffi-shim.c test/test-ffi-concurrency.c -pthread \ + -o "$(NATIVE_TEST_DIR)/test-ffi-concurrency-sanitize" + $(SANITIZER_CC) -std=c11 $(NATIVE_TEST_CFLAGS) $(SANITIZER_CFLAGS) \ + support/jsh-program-image.c test/test-program-image.c \ + -o "$(NATIVE_TEST_DIR)/test-program-image-sanitize" + ASAN_OPTIONS=detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 \ + "$(NATIVE_TEST_DIR)/test-ffi-concurrency-sanitize" + ASAN_OPTIONS=detect_leaks=0:halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 \ + "$(NATIVE_TEST_DIR)/test-program-image-sanitize" + +test-security-regressions: ensure-jerboa-tools $(FFI_LIB) jerboa + @REPO_ROOT="$(CURDIR)" JERBUILD="$(JERBUILD)" CC="$(CC)" \ + sh test/security-regressions.sh adversarial-corpus: ensure-jerboa-tools $(FFI_LIB) jerboa @echo "=== Running adversarial parser/redirection/job-control corpus ===" - $(JERBUILD) exec --libdirs "$(LIBDIRS)" support/adversarial-corpus-evidence.ss + $(JSH_FFI_ENV) $(JERBUILD) exec --libdirs "$(LIBDIRS)" support/adversarial-corpus-evidence.ss test-binary: @echo "=== Running binary smoke tests ===" @@ -132,7 +175,8 @@ timing-evidence: JSH_TIMING_EVIDENCE_DIR="$(TIMING_EVIDENCE_DIR)" \ sh tools/timing-evidence.sh -verify: security test adversarial-corpus audit timing-evidence sbom reproducibility-report +verify: security test test-native test-native-sanitize test-security-regressions \ + adversarial-corpus audit timing-evidence sbom reproducibility-report release-evidence: @rm -rf "$(RELEASE_EVIDENCE_DIR)" @@ -144,6 +188,11 @@ release-evidence: @$(MAKE) security > "$(RELEASE_EVIDENCE_DIR)/security.log" 2>&1 @echo "==> Running unit tests" @$(MAKE) test > "$(RELEASE_EVIDENCE_DIR)/test.log" 2>&1 + @echo "==> Running native concurrency and launcher tests" + @$(MAKE) test-native > "$(RELEASE_EVIDENCE_DIR)/test-native.log" 2>&1 + @$(MAKE) test-native-sanitize > "$(RELEASE_EVIDENCE_DIR)/test-native-sanitize.log" 2>&1 + @echo "==> Running loader and command-substitution security regressions" + @$(MAKE) test-security-regressions > "$(RELEASE_EVIDENCE_DIR)/test-security-regressions.log" 2>&1 @echo "==> Running adversarial corpus" @$(MAKE) adversarial-corpus > "$(RELEASE_EVIDENCE_DIR)/adversarial-corpus.log" 2>&1 @echo "==> Recording timing/fd lifecycle evidence status" @@ -169,11 +218,17 @@ release-evidence: bench: $(FFI_LIB) @echo "=== Running shell benchmarks ===" - @$(JERBUILD) exec --libdirs "$(LIBDIRS)" bench.ss - @$(JERBUILD) exec --libdirs "$(LIBDIRS)" bench-smp.ss + @$(JSH_FFI_ENV) $(JERBUILD) exec --libdirs "$(LIBDIRS)" bench.ss + @$(JSH_FFI_ENV) $(JERBUILD) exec --libdirs "$(LIBDIRS)" bench-smp.ss $(OILS_DIR): - git clone --depth 1 https://github.com/oils-for-unix/oils.git $(OILS_DIR) + @rm -rf "$(OILS_DIR)"; \ + git init -q "$(OILS_DIR)"; \ + git -C "$(OILS_DIR)" remote add origin "$(OILS_REPOSITORY)"; \ + git -C "$(OILS_DIR)" fetch -q --depth 1 origin "$(OILS_COMMIT)"; \ + test "$$(git -C "$(OILS_DIR)" rev-parse FETCH_HEAD)" = "$(OILS_COMMIT)"; \ + git -C "$(OILS_DIR)" checkout -q --detach "$(OILS_COMMIT)"; \ + test "$$(git -C "$(OILS_DIR)" rev-parse 'HEAD^{tree}')" = "$(OILS_TREE)" compat-smoke: $(OILS_DIR) $(SH_SPEC) $(OILS_DIR)/spec/smoke.test.sh $(BASH) $(JSH) @@ -229,6 +284,9 @@ help: @echo "Test:" @echo " make security Release security metadata and secret scan" @echo " make test Unit tests" + @echo " make test-native Native concurrency and launcher security tests" + @echo " make test-native-sanitize ASan/UBSan native boundary tests" + @echo " make test-security-regressions Loader and command-substitution tests" @echo " make test-binary Binary smoke tests" @echo " make audit Native FFI symbol/dependency audit" @echo " make timing-evidence Record target timing/fd lifecycle proof status" --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ fetches the configured Jerboa release tool. ```sh make test +make test-native +make test-security-regressions make audit make release-evidence make test-binary @@ -34,6 +36,18 @@ scripts live here so speed work stays close to the shell implementation. `make release-evidence` records unit tests, native FFI audit output, SBOM manifests, and reproducibility output under `dist/release-evidence/`. +Command substitution is drained concurrently and is limited to 8 MiB by +default. Set `JSH_COMMAND_SUB_MAX_BYTES` to a positive byte limit no greater +than 1 GiB when a workload needs a different bound. Exceeding the limit aborts +the expansion after the producer has been drained, rather than hanging it on a +full pipe. + +Interpreted development runs load `libjsh-ffi` only when +`JSH_FFI_DEV_NATIVE=1` and `JSH_FFI_LIB` names an absolute, non-symlink +directory. Standalone builds register the shim statically; neither mode +searches the current directory or a bare library name. Development loading +also rejects ambient DYLD/LD search overrides and verifies the JSH1 ABI canary. + ## History Interactive `jsh` writes normal line-oriented shell history to `$HISTFILE` --- a/SECURITY.md +++ b/SECURITY.md @@ -18,6 +18,16 @@ production-readiness tracker are complete. input cannot reshape native argument vectors. - Blocking native read/write/poll/wait/terminal/file operations are declared collect-safe where they can wait on host I/O or subprocess state. +- Standalone builds use registered FFI symbols. Interpreted development loading + requires an explicit absolute, non-symlink override and never searches `.` or + a bare library name. +- Command substitution drains while the producer runs and has an 8 MiB default + capture ceiling (`JSH_COMMAND_SUB_MAX_BYTES`, hard maximum 1 GiB). +- Wait, pipe, and bounded-read results are caller-owned rather than process-wide + scratch slots. +- Non-memfd launcher images use a random private directory, exclusive no-follow + creation, a held descriptor, and identity verification immediately before + loading. - Entrypoints call `ffi_ensure_std_fds` before Scheme initialization so fds 0/1/2 are repaired even in unusual launcher contexts. - History is plaintext by default and is chmoded to `0600` after writes. @@ -26,6 +36,10 @@ production-readiness tracker are complete. - `make release-evidence` records unit tests, native FFI audit output, SBOM manifests, generated-source reproducibility, FFI-shim reproducibility, and sanitized host-neutral release evidence. +- `make test-native`, `make test-native-sanitize`, and + `make test-security-regressions` cover concurrent native results, integer and + capacity boundaries, launcher symlink/replacement attacks, unsafe loader + candidates, and high-output command substitution. ## Sensitive Data @@ -50,8 +64,8 @@ Local release candidates must pass: - Finish adversarial parser/expander/redirection/job-control corpus tests. - Add hosted Linux and macOS CI evidence for the FFI shim and shell unit tests. -- Add sanitizer or equivalent native-boundary CI for `ffi-shim.c` and the - jerbuild launcher. +- Add hosted sanitizer or equivalent native-boundary CI for `ffi-shim.c` and + the jerbuild launcher (local targets are already part of `make verify`). - Archive release evidence for Chez/Jerboa, the C compiler, the native FFI shim, Python test harnesses, optional Oils spec fixtures, and full binary builds when `JERBOA_SHELL_REPRO_BINARY=1` is used. --- a/docs/ffi-boundary.md +++ b/docs/ffi-boundary.md @@ -6,14 +6,32 @@ limit operations. ## Current Controls -- The FFI library is loaded lazily from `JSH_FFI_LIB`, the current directory, or - the platform library search path. +- Standalone binaries use statically registered FFI symbols. Interpreted + development runs require `JSH_FFI_DEV_NATIVE=1` plus an absolute, + canonical `JSH_FFI_LIB` directory. The directory, final library, and every + ancestor must be owned by root or the invoking user and must not be + world-writable; production paths also reject group write. Privileged + processes ignore these environment selectors. Current-directory and bare-name dynamic + loading are rejected, as are inherited DYLD/LD search overrides. Both + static and development providers must expose the expected JSH1 ABI canary + and complete representative symbol set. - String inputs that cross C as C strings reject embedded NUL bytes before use. - Packed argv/env/fd lists reject SOH bytes, because SOH is the C split delimiter used by `packed_split`. +- The fork/exec helper allocates and parses argv, env, and retained-fd state in + the parent. It blocks signals in the calling thread across `fork`; its child + resets copied runtime handlers before restoring an empty mask and performs + only async-signal-safe setup before `execve` or `_exit`. - Potentially blocking native operations are declared `__collect_safe`. - C read/write helpers retry `EINTR` and handle short writes where partial I/O is valid. +- Pipe descriptors, wait statuses, and terminal dimensions are copied into + caller-owned bytevectors. + Bounded-read data is held by a per-operation result handle and freed by the + same caller, so concurrent calls cannot overwrite shared scratch storage. +- Command substitution starts a collect-safe drain before execution, retains at + most `JSH_COMMAND_SUB_MAX_BYTES` (8 MiB by default), and discards excess bytes + through EOF before reporting a distinct limit error. - `ffi_ensure_std_fds` opens `/dev/null` onto fd 0/1/2 when any standard descriptor is closed. - `make release-evidence` records native FFI audit output, FFI shim hashes, @@ -29,17 +47,21 @@ limit operations. into Scheme-managed bytevectors after returning. - File descriptors returned by FFI helpers are owned by the caller and must be closed or moved with `dup2` according to shell redirection state. +- A bounded-read result owns its byte buffer until + `ffi_read_result_free`. The Scheme wrapper uses `dynamic-wind` so the result + is freed on success and exceptions. - Process substitution FIFOs are owned by their cleanup thunks and should not be exposed as durable user files. ## Known Gaps -- Sanitizer builds are not yet part of CI. +- `make test-native-sanitize` runs UBSan on macOS and ASan+UBSan on Linux. + Hosted CI evidence is still required before release claims. - The argv/env ABI is still delimiter-packed instead of length-vector based. Current validation rejects delimiter bytes; a future ABI should use explicit lengths. -- The shell intentionally exposes `fork`, `execve`, `setsid`, process groups, - terminal modes, and fd manipulation. These are shell primitives, not daemon - APIs. +- The shell intentionally exposes a bounded fork/exec launcher, `execve`, + `setsid`, process groups, terminal modes, and fd manipulation. These are + shell primitives, not daemon APIs. - Full binary reproducibility is opt-in through `JERBOA_SHELL_REPRO_BINARY=1` and is still required for release candidates that ship `jsh-macos`. --- a/docs/release-evidence.md +++ b/docs/release-evidence.md @@ -7,6 +7,8 @@ The evidence bundle contains: - security gate output; - unit-test output; +- native pipe/wait/read concurrency, launcher, and sanitizer output; +- unsafe-loader and bounded concurrent command-substitution regression output; - deterministic adversarial parser/redirection/job-control corpus output; - timing/fd lifecycle target proof status under `timing-evidence/`; - native FFI shim audit output; --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -32,6 +32,15 @@ - Process substitutions use `0600` FIFOs and registered cleanup thunks. - Native read/write/poll/wait operations use EINTR-aware C loops and collect-safe Scheme FFI declarations. +- Native shim lookup never searches the current directory or a bare filename; + dynamic loading is an explicit development-only absolute-path override. +- Command substitution drains concurrently with execution and retains at most + the configured byte limit, preventing pipe-capacity deadlocks and unbounded + capture growth. +- The standalone launcher uses Linux memfd when available. Its portable + fallback creates an exclusive no-follow 0600 image in a random private 0700 + directory, holds the descriptor, and verifies owner, mode, size, device, and + inode immediately before Chez loads it. - `ffi_ensure_std_fds` repairs closed standard descriptors before the Scheme runtime starts. - Release evidence records the security gate, unit tests, native FFI audit @@ -58,6 +67,7 @@ status instead of leaving it implicit. - Startup-file and completion-hook behavior needs explicit release policy, tracked through the same target timing proof. -- Native shim sanitizer builds and hosted cross-platform CI are still required. +- Hosted cross-platform sanitizer CI is still required; local native regression + targets cover caller-owned wait/pipe/read state and launcher replacement. - Full standalone binary reproducibility evidence is still required for release candidates that ship a binary. --- a/expander.ss +++ b/expander.ss @@ -5,7 +5,7 @@ (export #t) (import :std/sugar :std/format - (only (compat gambit) thread-join!) + (only (compat gambit) thread-join! thread-yield!) (only-in :std/misc/string string-prefix? string-suffix? string-contains) (only-in :std/misc/string-more string-replace) ./pregexp-compat @@ -19,7 +19,7 @@ :jsh/arithmetic) ;;; --- Latin-1 to UTF-8 re-decode --- -;;; ffi-read-all-from-fd uses char-string which decodes bytes as Latin-1. +;;; ffi-read-all-from-fd decodes captured bytes as Latin-1. ;;; Re-interpret as UTF-8 when the bytes form valid UTF-8, so that multi-byte ;;; characters (e.g. from printf \xE2\x98\xA0) are correctly decoded. (def (latin1->utf8 s) @@ -2159,73 +2159,119 @@ ;; Execute a command and capture its stdout ;; Strips trailing newlines (per POSIX) +(def command-substitution-default-max-bytes (* 8 1024 1024)) +(def command-substitution-hard-max-bytes (* 1024 1024 1024)) + +(def (command-substitution-output-limit) + (let ([configured (getenv "JSH_COMMAND_SUB_MAX_BYTES")]) + (if (or (not configured) (string=? configured "")) + command-substitution-default-max-bytes + (let ([value (string->number configured)]) + (unless (and (integer? value) (exact? value) (> value 0) + (<= value command-substitution-hard-max-bytes)) + (error 'command-substitute + "JSH_COMMAND_SUB_MAX_BYTES must be an integer from 1 through 1073741824" + configured)) + value)))) + +(def (command-substitute-with-executor cmd env exec-fn limit) + (let-values ([(read-fd write-fd) (ffi-pipe-raw)]) + (let* ([port-fd (ffi-dup write-fd)] + [pipe-port + (if (>= port-fd 0) + (with-catch + (lambda (e) + (ffi-close-fd port-fd) + (ffi-close-fd read-fd) + (ffi-close-fd write-fd) + (raise e)) + (lambda () + (open-fd-output-port port-fd + (buffer-mode block) + (native-transcoder)))) + (begin + (ffi-close-fd read-fd) + (ffi-close-fd write-fd) + (error 'command-substitute + "unable to duplicate command-substitution pipe")))] + [reader-result (vector #f)] + [reader-error (vector #f)] + [reader-ready (vector #f)] + [reader + (with-catch + (lambda (e) + (with-catch (lambda (_) #!void) (lambda () (close-port pipe-port))) + (ffi-close-fd read-fd) + (ffi-close-fd write-fd) + (raise e)) + (lambda () + (spawn + (lambda () + (vector-set! reader-ready 0 #t) + (unwind-protect + (with-catch + (lambda (e) (vector-set! reader-error 0 e)) + (lambda () + (vector-set! reader-result 0 + (latin1->utf8 + (ffi-read-all-from-fd read-fd limit))))) + (ffi-close-fd read-fd))))))]) + (let wait-for-reader () + (unless (vector-ref reader-ready 0) + (thread-yield!) + (wait-for-reader))) + ;; Parameterization restores the Scheme port on every exit. The unwind + ;; cleanup closes both writer references before joining the reader, so a + ;; producer exception cannot strand the drain thread waiting for EOF. + (unwind-protect + (parameterize ([current-output-port pipe-port] + [*in-subshell* #t] + [*in-dquote-context* #f] + [*pipeline-stdin-fd* #f] + [*pipeline-stdout-fd* write-fd]) + (let ([sub-env (env-clone env)]) + (with-catch + (lambda (e) + (if (subshell-exit-exception? e) + (env-set-last-status! + env (subshell-exit-exception-status e)) + (raise e))) + (lambda () + (exec-fn cmd sub-env) + (env-set-last-status! + env (shell-environment-last-status sub-env)))))) + (begin + (with-catch (lambda (e) #!void) + (lambda () (force-output pipe-port))) + (with-catch (lambda (e) #!void) + (lambda () (close-port pipe-port))) + (ffi-close-fd write-fd) + (thread-join! reader))) + (let ([failure (vector-ref reader-error 0)]) + (when failure (raise failure))) + (string-trim-trailing-newlines (vector-ref reader-result 0))))) + (def (command-substitute cmd env) (profile-measure (string-append "command substitution: " cmd) (lambda () ;; Signal that a command substitution ran (for bare assignment $? tracking) (*command-sub-ran* #t) - (let ((exec-fn (*execute-input*))) + (let ([exec-fn (*execute-input*)] + [limit (command-substitution-output-limit)]) (if exec-fn - ;; Use jsh's own executor: redirect stdout to a pipe, run command, read output - (with-catch - (lambda (e) "") - (lambda () - (let-values (((read-fd write-fd) (ffi-pipe-raw))) - ;; Save real stdout fd and Gambit port - (let ((saved-fd (ffi-dup 1)) - (saved-port (current-output-port))) - ;; Redirect real fd 1 to pipe write end - (ffi-dup2 write-fd 1) - (ffi-close-fd write-fd) - ;; Dup fd 1 (pipe write end) to a fresh fd owned by the Chez port. - ;; This lets close-port flush+close the port fd without touching fd 1, - ;; so external child processes still inherit fd 1 as the pipe write end. - (let* ((port-fd (ffi-dup 1)) - (pipe-port (open-fd-output-port port-fd (buffer-mode block) (native-transcoder)))) - (current-output-port pipe-port) - ;; Execute in subshell context so exit doesn't terminate parent. - ;; Clone env so changes (aliases, variables) don't leak back. - ;; Clear pipeline fd params so execute-external won't override - ;; fd 0/1 with pipeline pipe fds — we already redirected fd 1 - ;; to the capture pipe above. - (let ((sub-env (env-clone env))) - (with-catch - (lambda (e) - (when (subshell-exit-exception? e) - (env-set-last-status! env (subshell-exit-exception-status e)))) - (lambda () - (parameterize ((*in-subshell* #t) - (*in-dquote-context* #f) - (*pipeline-stdin-fd* #f) - (*pipeline-stdout-fd* #f)) - (exec-fn cmd sub-env)) - ;; Normal completion: propagate $? from subshell - (env-set-last-status! env (shell-environment-last-status sub-env))))) - ;; Flush and close the pipe port - (force-output pipe-port) - (close-port pipe-port)) - ;; Restore real fd 1 and Gambit port - (ffi-dup2 saved-fd 1) - (ffi-close-fd saved-fd) - (current-output-port saved-port) - ;; Read the output from the pipe read end using raw read() - ;; (open-input-file "/dev/fd/N" blocks on empty pipes in Gambit) - ;; Note: ffi-read-all-from-fd returns Latin-1 decoded string; - ;; re-decode as UTF-8 for correct multi-byte character handling - (let ((output (latin1->utf8 (ffi-read-all-from-fd read-fd)))) - (ffi-close-fd read-fd) - (string-trim-trailing-newlines output)))))) + ;; The reader starts first and drains concurrently. Native storage is + ;; bounded, while excess data is discarded through EOF so the producer + ;; can never deadlock on a full pipe. + (command-substitute-with-executor cmd env exec-fn limit) ;; Fallback: use /bin/sh if executor not initialized - (with-catch - (lambda (e) "") - (lambda () - (let* ((port (open-input-process - (list path: "/bin/sh" - arguments: (list "-c" cmd) - environment: (env-exported-alist env)))) - (output (read-all-string port))) - (close-port port) - (string-trim-trailing-newlines output))))))))) + (let ([port (open-input-process + (list path: "/bin/sh" + arguments: (list "-c" cmd) + environment: (env-exported-alist env)))]) + (unwind-protect + (string-trim-trailing-newlines + (read-all-string-bounded port limit)) + (close-port port)))))))) ;;; --- Arithmetic substitution --- @@ -3118,13 +3164,25 @@ (loop (- i 1)) (substring str 0 (+ i 1))))) -(def (read-all-string port) - (let ((buf (open-output-string))) - (let loop () - (let ((ch (read-char port))) +(def (read-all-string-bounded port max-bytes) + (let ([buf (open-output-string)]) + (let loop ([bytes 0] [too-large? #f]) + (let ([ch (read-char port)]) (if (eof-object? ch) - (get-output-string buf) - (begin (display ch buf) (loop))))))) + (if too-large? + (error 'command-substitute + "command substitution output exceeds configured byte limit" + max-bytes) + (get-output-string buf)) + (if too-large? + ;; Continue consuming after the cap without growing either the + ;; output buffer or a byte counter. + (loop bytes #t) + (let* ([encoded (string->utf8 (string ch))] + [next (+ bytes (bytevector-length encoded))] + [over? (> next max-bytes)]) + (unless over? (display ch buf)) + (loop next over?)))))))) (def (append-map f lst) (apply append (map f lst))) --- a/ffi-shim.c +++ b/ffi-shim.c @@ -57,18 +57,33 @@ extern char **environ; -static int waitpid_status = 0; -static int pipe_fds[2] = {-1, -1}; -static unsigned char *read_buf = NULL; -static int read_buf_len = 0; -static int ws_col = 80; -static int ws_row = 24; static __thread struct termios saved_termios[2]; static __thread int saved_termios_valid[2] = {0, 0}; static volatile sig_atomic_t signal_flags[NSIG]; +#ifndef JSH_RAW_READ #define JSH_RAW_READ read +#endif +#ifndef JSH_RAW_WRITE #define JSH_RAW_WRITE write +#endif + +#define JSH_READ_OK 0 +#define JSH_READ_TOO_LARGE 1 +#define JSH_READ_IO_ERROR 2 +#define JSH_READ_NO_MEMORY 3 + +_Static_assert(sizeof(int) == 4, "jsh FFI bytevector ABI requires 32-bit int"); + +/* Native loader ABI canary: "JSH1". */ +uint32_t ffi_jsh_abi_version(void) { return 0x4a534831u; } + +struct jsh_read_result { + unsigned char *data; + size_t length; + int status; + int error_number; +}; static ssize_t jsh_read_syscall(int fd, void *buf, size_t count) { return JSH_RAW_READ(fd, buf, count); @@ -125,13 +140,19 @@ static void packed_free(char **items) { static int parse_keep_fds(const char *packed, int *out, int max_out) { int n = 0; char **items = packed_split(packed); - if (!items) return 0; - for (int i = 0; items[i] && n < max_out; i++) { + if (!items) { + errno = ENOMEM; + return -1; + } + for (int i = 0; items[i]; i++) { char *end = NULL; long fd = strtol(items[i], &end, 10); - if (end && *end == '\0' && fd >= 0 && fd <= INT_MAX) { - out[n++] = (int)fd; + if (n >= max_out || !end || *end != '\0' || fd < 0 || fd > INT_MAX) { + packed_free(items); + errno = EINVAL; + return -1; } + out[n++] = (int)fd; } packed_free(items); return n; @@ -145,11 +166,7 @@ static int should_keep_fd(int fd, const int *keep, int keep_count) { return 0; } -static void close_unneeded_fds(const char *packed_keep_fds) { - int keep[256]; - int keep_count = parse_keep_fds(packed_keep_fds, keep, 256); - long max_fd = sysconf(_SC_OPEN_MAX); - if (max_fd < 0 || max_fd > 65536) max_fd = 65536; +static void close_unneeded_fds(const int *keep, int keep_count, long max_fd) { for (int fd = 3; fd < max_fd; fd++) { if (!should_keep_fd(fd, keep, keep_count)) close(fd); } @@ -161,7 +178,6 @@ int ffi_close_fd(int fd) { return close(fd); } int ffi_mkfifo(const char *path, int mode) { return mkfifo(path, (mode_t)mode); } int ffi_unlink(const char *path) { return unlink(path); } int ffi_chmod(const char *path, int mode) { return chmod(path, (mode_t)mode); } -int ffi_fork(void) { return (int)fork(); } void ffi_exit(int code) { _exit(code); } int ffi_getpid(void) { return (int)getpid(); } int ffi_getppid(void) { return (int)getppid(); } @@ -212,27 +228,32 @@ long ffi_lseek_end(int fd) { return (long)lseek(fd, 0, SEEK_END); } -int ffi_do_pipe(void) { - if (pipe(pipe_fds) != 0) { - pipe_fds[0] = -1; - pipe_fds[1] = -1; +int ffi_pipe_pair(unsigned char *out, int out_len) { + int fds[2] = {-1, -1}; + if (!out || out_len < (int)sizeof(fds)) { + errno = EINVAL; return -1; } + if (pipe(fds) != 0) return -1; + memcpy(out, fds, sizeof(fds)); return 0; } -int ffi_pipe_read_fd(void) { return pipe_fds[0]; } -int ffi_pipe_write_fd(void) { return pipe_fds[1]; } - -int ffi_do_waitpid(int pid, int options) { - pid_t r = waitpid((pid_t)pid, &waitpid_status, options); +int ffi_waitpid_result(int pid, int options, + unsigned char *status_out, int status_out_len) { + int status = 0; + pid_t r; + if (!status_out || status_out_len < (int)sizeof(status)) { + errno = EINVAL; + return -1; + } + do { + r = waitpid((pid_t)pid, &status, options); + } while (r < 0 && errno == EINTR); + if (r > 0) memcpy(status_out, &status, sizeof(status)); return (int)r; } -int ffi_get_waitpid_status(void) { - return waitpid_status; -} - int ffi_do_execve(const char *path, const char *packed_argv, const char *packed_env) { char **argv = packed_split(packed_argv); char **envp = packed_split(packed_env); @@ -254,37 +275,124 @@ int ffi_fork_exec(const char *path, int pgid, const char *packed_keep_fds, const char *cwd) { + int keep[256]; + int keep_count; + long max_fd; + char **argv; + char **envp; + struct sigaction default_action; + unsigned char reset_signal[NSIG]; + sigset_t all_signal_mask; + sigset_t empty_signal_mask; + sigset_t parent_signal_mask; + + if (!path || !packed_argv || !packed_env) { + errno = EINVAL; + return -1; + } + + /* Everything that may allocate or consult process-global libc state is + completed in the parent. The post-fork child is restricted to + async-signal-safe setup plus execve/_exit. */ + keep_count = parse_keep_fds(packed_keep_fds, keep, 256); + if (keep_count < 0) return -1; + max_fd = sysconf(_SC_OPEN_MAX); + if (max_fd < 0 || max_fd > 65536) max_fd = 65536; + argv = packed_split(packed_argv); + envp = packed_split(packed_env); + if (!argv || !envp) { + packed_free(argv); + packed_free(envp); + errno = ENOMEM; + return -1; + } + memset(&default_action, 0, sizeof(default_action)); + memset(reset_signal, 0, sizeof(reset_signal)); + default_action.sa_handler = SIG_DFL; + if (sigemptyset(&default_action.sa_mask) != 0 || + sigemptyset(&empty_signal_mask) != 0 || + sigfillset(&all_signal_mask) != 0) { + int saved_errno = errno; + packed_free(argv); + packed_free(envp); + errno = saved_errno; + return -1; + } + + /* Record which ignored dispositions must survive exec (for example a + nohup-inherited SIGHUP). Reset every other catchable signal in the + child, including signals that are currently default, so a concurrent + default-to-runtime-handler change cannot escape this snapshot. */ + for (int signum = 1; signum < NSIG; signum++) { + struct sigaction inherited_action; + if (signum != SIGKILL && signum != SIGSTOP && + sigaction(signum, NULL, &inherited_action) == 0 && + inherited_action.sa_handler != SIG_IGN) { + reset_signal[signum] = 1; + } + } + reset_signal[SIGINT] = 1; + reset_signal[SIGQUIT] = 1; + reset_signal[SIGTERM] = 1; + reset_signal[SIGPIPE] = 1; + reset_signal[SIGTSTP] = 1; + reset_signal[SIGTTIN] = 1; + reset_signal[SIGTTOU] = 1; + + /* Block signals in the calling thread across fork. Otherwise a copied + Scheme signal handler could run before the C child reaches execve. */ + if (sigprocmask(SIG_SETMASK, &all_signal_mask, &parent_signal_mask) != 0) { + int saved_errno = errno; + packed_free(argv); + packed_free(envp); + errno = saved_errno; + return -1; + } + pid_t pid = fork(); - if (pid < 0) return -1; + if (pid < 0) { + int saved_errno = errno; + (void)sigprocmask(SIG_SETMASK, &parent_signal_mask, NULL); + packed_free(argv); + packed_free(envp); + errno = saved_errno; + return -1; + } if (pid == 0) { + for (int signum = 1; signum < NSIG; signum++) { + if (reset_signal[signum] && + sigaction(signum, &default_action, NULL) != 0) _exit(126); + } + if (pgid == -1) { setpgid(0, 0); } else if (pgid > 0) { setpgid(0, (pid_t)pgid); } - signal(SIGINT, SIG_DFL); - signal(SIGQUIT, SIG_DFL); - signal(SIGTERM, SIG_DFL); - signal(SIGPIPE, SIG_DFL); - signal(SIGTSTP, SIG_DFL); - signal(SIGTTIN, SIG_DFL); - signal(SIGTTOU, SIG_DFL); - if (cwd && cwd[0] != '\0') { if (chdir(cwd) != 0) _exit(126); } - close_unneeded_fds(packed_keep_fds); - - char **argv = packed_split(packed_argv); - char **envp = packed_split(packed_env); - if (!argv || !envp) _exit(126); + close_unneeded_fds(keep, keep_count, max_fd); + if (sigprocmask(SIG_SETMASK, &empty_signal_mask, NULL) != 0) _exit(126); execve(path, argv, envp); _exit(errno == ENOENT ? 127 : 126); } + if (sigprocmask(SIG_SETMASK, &parent_signal_mask, NULL) != 0) { + int saved_errno = errno; + (void)kill(pid, SIGKILL); + while (waitpid(pid, NULL, 0) < 0 && errno == EINTR) { } + packed_free(argv); + packed_free(envp); + errno = saved_errno; + return -1; + } + packed_free(argv); + packed_free(envp); + if (pgid == -1) { setpgid(pid, pid); } else if (pgid > 0) { @@ -308,41 +416,166 @@ const char *ffi_environ_entry(int index) { return ""; } -int ffi_do_read_all(int fd) { - free(read_buf); - read_buf = NULL; - read_buf_len = 0; +static int jsh_next_capacity(size_t current, size_t limit, size_t *next_out) { + size_t next; + if (!next_out || current >= limit) return 0; + if (current > SIZE_MAX / 2) { + next = limit; + } else { + next = current * 2; + if (next > limit) next = limit; + } + if (next <= current) return 0; + *next_out = next; + return 1; +} + +#ifdef JSH_FFI_TESTING +int jsh_test_next_capacity(unsigned long long current, + unsigned long long limit, + unsigned long long *next_out) { + size_t next = 0; + if (!next_out || current > (unsigned long long)SIZE_MAX || + limit > (unsigned long long)SIZE_MAX || + !jsh_next_capacity((size_t)current, (size_t)limit, &next)) return 0; + *next_out = (unsigned long long)next; + return 1; +} +#endif - size_t cap = 4096; - read_buf = (unsigned char *)malloc(cap); - if (!read_buf) return 0; +static void jsh_drain_fd(int fd, int *status, int *error_number) { + unsigned char discard[8192]; + for (;;) { + ssize_t n; + do { + n = jsh_read_syscall(fd, discard, sizeof(discard)); + } while (n < 0 && errno == EINTR); + if (n > 0) continue; + if (n < 0 && *status == JSH_READ_OK) { + *status = JSH_READ_IO_ERROR; + *error_number = errno; + } + return; + } +} + +void *ffi_read_all_bounded(int fd, long long max_bytes) { + struct jsh_read_result *result; + size_t limit; + size_t cap; + + result = (struct jsh_read_result *)calloc(1, sizeof(*result)); + if (!result) return NULL; + result->status = JSH_READ_OK; + + if (max_bytes <= 0 || (unsigned long long)max_bytes > (unsigned long long)SIZE_MAX) { + result->status = JSH_READ_TOO_LARGE; + jsh_drain_fd(fd, &result->status, &result->error_number); + return result; + } + limit = (size_t)max_bytes; + cap = limit < 4096 ? limit : 4096; + result->data = (unsigned char *)malloc(cap); + if (!result->data) { + result->status = JSH_READ_NO_MEMORY; + jsh_drain_fd(fd, &result->status, &result->error_number); + return result; + } for (;;) { - if ((size_t)read_buf_len == cap) { - cap *= 2; - unsigned char *next = (unsigned char *)realloc(read_buf, cap); - if (!next) break; - read_buf = next; + if (result->length == cap) { + if (cap == limit) { + unsigned char extra; + ssize_t n; + do { + n = jsh_read_syscall(fd, &extra, 1); + } while (n < 0 && errno == EINTR); + if (n > 0) { + result->status = JSH_READ_TOO_LARGE; + jsh_drain_fd(fd, &result->status, &result->error_number); + } else if (n < 0) { + result->status = JSH_READ_IO_ERROR; + result->error_number = errno; + } + break; + } + size_t next_cap = 0; + if (!jsh_next_capacity(cap, limit, &next_cap)) { + result->status = JSH_READ_TOO_LARGE; + jsh_drain_fd(fd, &result->status, &result->error_number); + break; + } + unsigned char *next = (unsigned char *)realloc(result->data, next_cap); + if (!next) { + result->status = JSH_READ_NO_MEMORY; + jsh_drain_fd(fd, &result->status, &result->error_number); + break; + } + result->data = next; + cap = next_cap; } + ssize_t n; do { - n = jsh_read_syscall(fd, read_buf + read_buf_len, cap - (size_t)read_buf_len); + n = jsh_read_syscall(fd, result->data + result->length, + cap - result->length); } while (n < 0 && errno == EINTR); if (n > 0) { - read_buf_len += (int)n; + if ((size_t)n > cap - result->length) { + result->status = JSH_READ_IO_ERROR; + result->error_number = EOVERFLOW; + jsh_drain_fd(fd, &result->status, &result->error_number); + break; + } + result->length += (size_t)n; continue; } - if (n == 0) break; + if (n < 0) { + result->status = JSH_READ_IO_ERROR; + result->error_number = errno; + } break; } - return read_buf_len; + return result; } -int ffi_copy_read_buf(unsigned char *out, int max_out) { - if (!out || max_out <= 0 || !read_buf) return 0; - int n = read_buf_len < max_out ? read_buf_len : max_out; - memcpy(out, read_buf, (size_t)n); - return n; +int ffi_read_result_status(void *opaque) { + const struct jsh_read_result *result = (const struct jsh_read_result *)opaque;