Resolve security audit findings
ober
2435abe9974589e3f811d68a4cf48eb6662c4c85
--- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: build-test-audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install system tools run: | --- a/.github/workflows/security-baseline.yml +++ b/.github/workflows/security-baseline.yml @@ -13,7 +13,7 @@ jobs: baseline: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Required release files run: | --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ JERBOA_VERSION ?= v0.2.3 JERBOA_TOOL_DIR ?= $(CURDIR)/.jerboa/bin JERBUILD ?= $(shell if [ -x "$(CURDIR)/../jerboa/dist/jerbuild" ]; then printf '%s\n' "$(CURDIR)/../jerboa/dist/jerbuild"; elif command -v jerbuild >/dev/null 2>&1; then command -v jerbuild; else printf '%s\n' "$(JERBOA_TOOL_DIR)/jerbuild"; fi) JH = $(shell "$(JERBUILD)" --jerboa-home 2>/dev/null) -LIBDIRS = lib:$(JH)/lib +JERBOA_CORE ?= $(CURDIR)/../jerboa +LIBDIRS = lib:$(JH)/lib:$(JERBOA_CORE)/lib PCRE2_CFLAGS ?= $(shell pkg-config --cflags libpcre2-8 2>/dev/null) PCRE2_LIBS ?= $(shell pkg-config --libs libpcre2-8 2>/dev/null || printf '%s\n' -lpcre2-8) @@ -28,8 +29,20 @@ else endif SHIM := jerboa_pcre2_shim.$(SHLIB_EXT) +NATIVE_TEST_DIR := .jerboa/native-tests +NATIVE_TEST := $(NATIVE_TEST_DIR)/pcre2-native-concurrency +NATIVE_UBSAN_TEST := $(NATIVE_TEST_DIR)/pcre2-native-ubsan +NATIVE_ASAN_TEST := $(NATIVE_TEST_DIR)/pcre2-native-asan +NATIVE_TSAN_TEST := $(NATIVE_TEST_DIR)/pcre2-native-tsan +ifeq ($(UNAME_S),Darwin) + SANITIZER_THREADS := 2 + SANITIZER_ITERATIONS := 2 +else + SANITIZER_THREADS := 4 + SANITIZER_ITERATIONS := 250 +endif -.PHONY: all build transpile test hostile-regex-corpus clean shim ensure-jerboa-tools security audit pcre2-advisory-check sbom reproducibility-report target-evidence verify release-evidence +.PHONY: all build transpile test native-test native-sanitizer-test loader-policy-test hostile-regex-corpus clean shim ensure-jerboa-tools security audit pcre2-advisory-check sbom reproducibility-report target-evidence verify release-evidence all: build @@ -49,15 +62,59 @@ transpile: ensure-jerboa-tools build: ensure-jerboa-tools shim transpile -test: build - @JERBOA_PCRE2_LIB=$(CURDIR) \ +test: build native-test loader-policy-test + @JERBOA_PCRE2_DEV_NATIVE=1 JERBOA_PCRE2_LIB=$(CURDIR) \ DYLD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$DYLD_LIBRARY_PATH \ LD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$LD_LIBRARY_PATH \ $(JERBUILD) exec --libdirs "$(LIBDIRS)" tests/pcre2-test.ss +native-test: tests/pcre2-native-concurrency.c jerboa_pcre2_shim.c + @mkdir -p "$(NATIVE_TEST_DIR)" + $(CC) $(CFLAGS) $(PCRE2_CFLAGS) -pthread -o "$(NATIVE_TEST)" \ + tests/pcre2-native-concurrency.c jerboa_pcre2_shim.c $(PCRE2_LIBS) + @"$(NATIVE_TEST)" + +native-sanitizer-test: tests/pcre2-native-concurrency.c jerboa_pcre2_shim.c + @mkdir -p "$(NATIVE_TEST_DIR)" + $(CC) -O1 -g -Wall -Wextra -Werror -fno-omit-frame-pointer \ + -DJPCRE2_TEST_THREADS=$(SANITIZER_THREADS) \ + -DJPCRE2_TEST_ITERATIONS=$(SANITIZER_ITERATIONS) \ + -fsanitize=undefined $(PCRE2_CFLAGS) -pthread \ + -o "$(NATIVE_UBSAN_TEST)" tests/pcre2-native-concurrency.c \ + jerboa_pcre2_shim.c $(PCRE2_LIBS) + @UBSAN_OPTIONS=halt_on_error=1 "$(NATIVE_UBSAN_TEST)" + $(CC) -O1 -g -Wall -Wextra -Werror -fno-omit-frame-pointer \ + -DJPCRE2_TEST_THREADS=$(SANITIZER_THREADS) \ + -DJPCRE2_TEST_ITERATIONS=$(SANITIZER_ITERATIONS) \ + -fsanitize=address,undefined \ + $(PCRE2_CFLAGS) -pthread \ + -o "$(NATIVE_ASAN_TEST)" tests/pcre2-native-concurrency.c \ + jerboa_pcre2_shim.c $(PCRE2_LIBS) + $(CC) -O1 -g -Wall -Wextra -Werror -fno-omit-frame-pointer \ + -DJPCRE2_TEST_THREADS=$(SANITIZER_THREADS) \ + -DJPCRE2_TEST_ITERATIONS=$(SANITIZER_ITERATIONS) \ + -fsanitize=thread \ + $(PCRE2_CFLAGS) -pthread \ + -o "$(NATIVE_TSAN_TEST)" tests/pcre2-native-concurrency.c \ + jerboa_pcre2_shim.c $(PCRE2_LIBS) + @if [ "$(UNAME_S)" = Darwin ]; then \ + echo "sanitizer-runtime: ASan/TSan execution skipped on Darwin; binaries compiled, run this target on Linux"; \ + else \ + ASAN_OPTIONS=halt_on_error=1 UBSAN_OPTIONS=halt_on_error=1 \ + "$(NATIVE_ASAN_TEST)"; \ + TSAN_OPTIONS=halt_on_error=1 "$(NATIVE_TSAN_TEST)"; \ + fi + +loader-policy-test: build + @REPO_ROOT="$(CURDIR)" JERBUILD="$(JERBUILD)" \ + LIBDIRS="$(CURDIR)/lib:$(JH)/lib:$(JERBOA_CORE)/lib" SHIM="$(CURDIR)/$(SHIM)" \ + DYLD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$DYLD_LIBRARY_PATH \ + LD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$LD_LIBRARY_PATH \ + sh tests/loader-policy-test.sh + hostile-regex-corpus: build @mkdir -p dist - @JERBOA_PCRE2_LIB=$(CURDIR) \ + @JERBOA_PCRE2_DEV_NATIVE=1 JERBOA_PCRE2_LIB=$(CURDIR) \ DYLD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$DYLD_LIBRARY_PATH \ LD_LIBRARY_PATH=$(PCRE2_LIBDIR):$(CURDIR):$$LD_LIBRARY_PATH \ $(JERBUILD) exec --libdirs "$(LIBDIRS)" support/hostile-regex-corpus.ss > dist/hostile-regex-corpus.txt @@ -107,7 +164,7 @@ reproducibility-report: target-evidence: scripts/target-evidence.sh @REPO_ROOT="$(CURDIR)" TARGET_EVIDENCE_DIR="$(TARGET_EVIDENCE_DIR)" sh scripts/target-evidence.sh -verify: security test hostile-regex-corpus audit pcre2-advisory-check sbom reproducibility-report target-evidence +verify: security native-sanitizer-test test hostile-regex-corpus audit pcre2-advisory-check sbom reproducibility-report target-evidence release-evidence: security test hostile-regex-corpus audit sbom target-evidence rm -rf "$(DIST_DIR)" @@ -149,4 +206,4 @@ release-evidence: security test hostile-regex-corpus audit sbom target-evidence clean: rm -f jerboa_pcre2_shim.so jerboa_pcre2_shim.dylib pcre2_shim.so pcre2_shim.dylib - rm -rf lib .jerboa/bin + rm -rf lib .jerboa/bin "$(NATIVE_TEST_DIR)" --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ sudo apt-get install build-essential pkg-config libpcre2-dev ```sh make test +make native-sanitizer-test make hostile-regex-corpus make audit make pcre2-advisory-check @@ -32,8 +33,26 @@ make release-evidence ``` `make test` builds `jerboa_pcre2_shim`, transpiles the Jerboa libraries, and -runs the PCRE2 regression tests. If `jerbuild` is not installed, the Makefile -bootstraps the pinned Jerboa release into `.jerboa/bin`. +runs the PCRE2 regression, concurrent lifecycle, and native-loader policy +tests. If `jerbuild` is not installed, the Makefile bootstraps the pinned +Jerboa release into `.jerboa/bin`. + +Dynamic development builds must explicitly set `JERBOA_PCRE2_DEV_NATIVE=1` +and set `JERBOA_PCRE2_LIB` to an absolute trusted directory containing +`jerboa_pcre2_shim.so` or `jerboa_pcre2_shim.dylib`. Relative paths, symlinked +directory components, unexpected owners, world-writable paths, disallowed +group-writable paths, current-directory lookup, and bare-name lookup are +rejected by the shared `(std native-loader)` policy. Explicit development mode +permits group-write only for components owned by the same unprivileged user. +Privileged processes ignore both development variables. Production builds +should statically link the shim; +statically linked hosts are detected from their registered PCRE2 shim symbol +and do not need either environment variable. + +`make native-sanitizer-test` runs UBSan everywhere and ASan/TSan on Linux. +The current Apple sanitizer runtimes do not enter `main` reliably on this +macOS release, so Darwin builds compile those instrumented binaries and print +an explicit skip; Linux CI remains the required ASan/TSan execution gate. `make hostile-regex-corpus` runs an expanded deterministic adversarial-pattern corpus under low PCRE2 match/depth limits and records fail-closed behavior for @@ -71,6 +90,9 @@ hostnames, or raw command output. PCRE2 is a backtracking engine. For untrusted patterns, set conservative match limits with `(pcre2-set-match-limits! match-limit depth-limit)` and use an admission/review step before running user-authored regular expressions. +Compiled regex objects may be shared between threads: match data and +substitution results are per operation, and `pcre2-release!` synchronizes with +in-flight native calls. New calls fail after release. ## API --- a/SECURITY.md +++ b/SECURITY.md @@ -56,10 +56,22 @@ The FFI boundary is documented in `docs/ffi-boundary.md`. Security-sensitive expectations: - Dynamic native loading is lazy and must not run at library import time. +- Dynamic loading is an explicit development-only override requiring + `JERBOA_PCRE2_DEV_NATIVE=1` and a trusted absolute `JERBOA_PCRE2_LIB` + directory. The shared `(std native-loader)` policy validates the canonical + library and every ancestor's type, owner, and write mode; relative, + symlink-component, unexpected-owner, world-writable, current-directory, and + bare-name lookup fail closed. Group-write is accepted only in explicit + development mode when that component belongs to the same unprivileged user. + Privileged processes ignore the override. + Production builds should statically link the shim. - Blocking or CPU-heavy native calls use collect-safe FFI declarations. -- Match-data and compiled-pattern handles must be freed on success, no-match, - and exception paths. -- Released regex objects must not be reusable. +- Match data and substitution results are operation-owned and freed on success, + no-match, and exception paths. +- Each compiled regex has a lifecycle guard held from validation through the + final native use. Release uses the same guard, is idempotent, and makes all + subsequent operations fail before C is called. +- The bounded process-global pattern cache is synchronized. - C entry points reject null pointers, invalid offsets, and unsafe allocation sizes. --- a/docs/ffi-boundary.md +++ b/docs/ffi-boundary.md @@ -10,20 +10,40 @@ returning. - Native loading is lazy through `(jerboa ffi)` so importing the module does not crash static binaries or feature probes. +- Dynamic loading is a development-only override requiring + `JERBOA_PCRE2_DEV_NATIVE=1` and an absolute, non-symlinked trusted directory + in `JERBOA_PCRE2_LIB`. The shared `(std native-loader)` policy validates the + canonical library and every ancestor's type, owner, and write mode. No + current-directory or bare-name fallback exists, and privileged processes + ignore environment overrides. World-write always fails; group-write is + allowed only for same-user components in explicit unprivileged development + mode. Production hosts should statically link the shim. - Matching, JIT matching, compilation, and substitution use collect-safe FFI declarations because they can consume CPU or wait in native code. ## Memory And Lifetime Rules -- `pcre2-compile` owns a `pcre2_code` pointer and one reusable match-data handle. -- Per-call match-data allocated by search/match/fold/split helpers is freed on - success, no-match, and exception paths. -- `pcre2-release!` frees native handles, zeros the record fields, removes the - regex from the cache, and makes later reuse raise before C is called. +- `pcre2-compile` owns one `pcre2_code` pointer. Every match and substitution + allocates its own match-data handle; no mutable match data is stored on the + compiled regex. +- Per-call match data and substitution results are freed on success, no-match, + and exception paths. +- Every native use holds the compiled regex's lifecycle mutex from its live + check through its final pointer access. `pcre2-release!` takes that same + mutex, marks the pointer closed before freeing it, removes the object under a + separate cache mutex, and is safe to repeat. Later reuse raises before C is + called. +- The process-global LRU cache is bounded at 64 entries and all lookup, + promotion, insertion, eviction, and removal operations hold its mutex. - Result BLOB-like arbitrary bytes are not a goal of the string substitution API; outputs are treated as UTF-8 strings. - C entry points reject null pointers, invalid offsets, and unsafe allocation sizes as a second line of defense. +- `make test` includes barrier-synchronized replace/match/cache/release stress + and fresh-process relative/symlink/current-directory loader rejection tests. +- `make native-sanitizer-test` runs the native concurrency harness under + UBSan and, on Linux, ASan and TSan. Darwin records an explicit ASan/TSan + runtime skip after compiling the instrumented binaries. ## ReDoS And Resource Limits --- a/jerboa_pcre2_shim.c +++ b/jerboa_pcre2_shim.c @@ -153,87 +153,130 @@ size_t jerboa_pcre2_get_startchar(pcre2_match_data_8* md) } /* ------------------------------------------------------------------- - * Substitute — result buffer is thread-local. + * Substitute — each operation owns its result. * ------------------------------------------------------------------- */ -static __thread char* _ffi_subst_buf = NULL; -static __thread size_t _ffi_subst_len = 0; +typedef struct jerboa_pcre2_substitute_result { + int code; + size_t length; + char *data; +} jerboa_pcre2_substitute_result_t; -int jerboa_pcre2_do_substitute( +jerboa_pcre2_substitute_result_t* jerboa_pcre2_substitute_create( const pcre2_code_8* code, const char* subject, size_t subject_length, size_t startoffset, uint32_t options, pcre2_match_data_8* match_data, const char* replacement, size_t replacement_length) { + jerboa_pcre2_substitute_result_t *result; + pcre2_match_context_8 *ctx = NULL; + size_t capacity; size_t outlen; uint32_t opts; int rc; + result = (jerboa_pcre2_substitute_result_t*)calloc(1, sizeof(*result)); + if (!result) return NULL; + if (!code || (!subject && subject_length > 0) || (!replacement && replacement_length > 0) || !match_data) { - return PCRE2_ERROR_NULL; + result->code = PCRE2_ERROR_NULL; + return result; + } + if (startoffset > subject_length) { + result->code = PCRE2_ERROR_BADOFFSET; + return result; } - if (startoffset > subject_length) return PCRE2_ERROR_BADOFFSET; if (subject_length > SIZE_MAX - replacement_length || - subject_length + replacement_length > SIZE_MAX - 256) { - return PCRE2_ERROR_NOMEMORY; + subject_length + replacement_length >= SIZE_MAX - 256) { + result->code = PCRE2_ERROR_NOMEMORY; + return result; } - if (_ffi_subst_buf) { free(_ffi_subst_buf); _ffi_subst_buf = NULL; } - _ffi_subst_len = 0; + ctx = jerboa_pcre2_make_match_context(); + if (!ctx) { + result->code = PCRE2_ERROR_NOMEMORY; + return result; + } - outlen = subject_length + replacement_length + 256; - _ffi_subst_buf = (char*)malloc(outlen); - if (!_ffi_subst_buf) return PCRE2_ERROR_NOMEMORY; + capacity = subject_length + replacement_length + 256; + result->data = (char*)malloc(capacity + 1); + if (!result->data) { + pcre2_match_context_free_8(ctx); + result->code = PCRE2_ERROR_NOMEMORY; + return result; + } opts = options | PCRE2_SUBSTITUTE_OVERFLOW_LENGTH; + outlen = capacity; rc = pcre2_substitute_8( code, (PCRE2_SPTR8)subject, subject_length, startoffset, opts, - match_data, NULL, + match_data, ctx, (PCRE2_SPTR8)replacement, replacement_length, - (PCRE2_UCHAR8*)_ffi_subst_buf, &outlen); + (PCRE2_UCHAR8*)result->data, &outlen); if (rc == PCRE2_ERROR_NOMEMORY) { - free(_ffi_subst_buf); + free(result->data); + result->data = NULL; if (outlen == SIZE_MAX) { - _ffi_subst_buf = NULL; - return PCRE2_ERROR_NOMEMORY; + rc = PCRE2_ERROR_NOMEMORY; + } else { + capacity = outlen; + result->data = (char*)malloc(capacity + 1); + if (!result->data) { + rc = PCRE2_ERROR_NOMEMORY; + } else { + outlen = capacity; + rc = pcre2_substitute_8( + code, + (PCRE2_SPTR8)subject, subject_length, + startoffset, opts, + match_data, ctx, + (PCRE2_SPTR8)replacement, replacement_length, + (PCRE2_UCHAR8*)result->data, &outlen); + } } - _ffi_subst_buf = (char*)malloc(outlen + 1); - if (!_ffi_subst_buf) return PCRE2_ERROR_NOMEMORY; - size_t outlen2 = outlen + 1; - rc = pcre2_substitute_8( - code, - (PCRE2_SPTR8)subject, subject_length, - startoffset, opts, - match_data, NULL, - (PCRE2_SPTR8)replacement, replacement_length, - (PCRE2_UCHAR8*)_ffi_subst_buf, &outlen2); - if (rc >= 0) _ffi_subst_len = outlen2; - } else if (rc >= 0) { - _ffi_subst_len = outlen; } - if (rc < 0) { free(_ffi_subst_buf); _ffi_subst_buf = NULL; _ffi_subst_len = 0; } - return rc; + pcre2_match_context_free_8(ctx); + result->code = rc; + if (rc >= 0) { + result->length = outlen; + result->data[outlen] = '\0'; + } else { + free(result->data); + result->data = NULL; + result->length = 0; + } + return result; } -const char* jerboa_pcre2_substitute_result(void) +int jerboa_pcre2_substitute_code(const jerboa_pcre2_substitute_result_t *result) { - return _ffi_subst_buf ? _ffi_subst_buf : ""; + return result ? result->code : PCRE2_ERROR_NULL; } -size_t jerboa_pcre2_substitute_result_length(void) +const char* jerboa_pcre2_substitute_result( + const jerboa_pcre2_substitute_result_t *result) { - return _ffi_subst_len; + return result && result->data ? result->data : ""; } -int jerboa_pcre2_substitute_free(void) +size_t jerboa_pcre2_substitute_result_length( + const jerboa_pcre2_substitute_result_t *result) { - if (_ffi_subst_buf) { free(_ffi_subst_buf); _ffi_subst_buf = NULL; } - _ffi_subst_len = 0; + return result ? result->length : 0; +} + +int jerboa_pcre2_substitute_free(jerboa_pcre2_substitute_result_t *result) +{ + if (result) { + free(result->data); + result->data = NULL; + free(result); + } return 0; } --- a/scripts/security-check.sh +++ b/scripts/security-check.sh @@ -33,6 +33,8 @@ require_file scripts/reproducibility-report.sh require_file scripts/target-evidence.sh require_file scripts/sanitize-evidence.sh require_file support/hostile-regex-corpus.ss +require_file tests/loader-policy-test.sh +require_file tests/loader-policy-test.ss if ! grep -q 'JPCRE2_TARGET_PROOF_FILE' .jerboa/security.json || ! grep -q 'JPCRE2_TARGET_PROOF_FILE' SECURITY.md docs/release-evidence.md README.md || @@ -61,6 +63,30 @@ if ! rg -q 'load-shared-object[*]' src || fail=1 fi +check_empty "current-directory or bare-name PCRE2 shim loading" \ + rg -n -e '"\./jerboa_pcre2_shim' -e 'load-shared-object[*][[:space:]]+"jerboa_pcre2_shim' \ + src + +if ! grep -q '(std native-loader)' src/jerboa-pcre2/ffi.ss || + ! grep -q 'native-loader-development-enabled?' src/jerboa-pcre2/ffi.ss || + ! grep -q 'native-loader-validate-directory!' src/jerboa-pcre2/ffi.ss || + ! grep -q 'native-loader-validate-library!' src/jerboa-pcre2/ffi.ss || + ! grep -q 'native-loader-privileged?' src/jerboa-pcre2/ffi.ss || + ! grep -q 'foreign-entry? "jerboa_pcre2_compile"' src/jerboa-pcre2/ffi.ss || + ! grep -q 'world-writable-ancestor' tests/loader-policy-test.sh || + ! grep -q 'privileged-env-ignored' tests/loader-policy-test.sh || + ! grep -q 'loader-policy-test' Makefile; then + printf '%s\n' "shared native-loader policy or hostile-path coverage is missing" >&2 + fail=1 +fi + +if rg -q 'pcre-regex-match-data' src/jerboa-pcre2/pcre2.ss || + ! grep -q 'with-live-regex' src/jerboa-pcre2/pcre2.ss || + ! grep -q '\*cache-lock\*' src/jerboa-pcre2/pcre2.ss; then + printf '%s\n' "per-operation match data or synchronized lifecycle/cache guard is missing" >&2 + fail=1 +fi + if ! grep -q 'pcre2_set_match_limit_8' jerboa_pcre2_shim.c || ! grep -q 'pcre2_set_depth_limit_8' jerboa_pcre2_shim.c || ! grep -q 'pcre2_match_context_free_8' jerboa_pcre2_shim.c; then --- a/src/jerboa-pcre2/ffi.ss +++ b/src/jerboa-pcre2/ffi.ss @@ -35,7 +35,8 @@ ffi-pcre2-match-data-free ;; Substitute - ffi-pcre2-do-substitute + ffi-pcre2-substitute-create + ffi-pcre2-substitute-code ffi-pcre2-substitute-result ffi-pcre2-substitute-result-length ffi-pcre2-substitute-free @@ -54,7 +55,15 @@ ffi-pcre2-set-match-limits) (import (jerboa prelude) - (only (jerboa ffi) c-lambda load-shared-object*)) + (only (jerboa ffi) c-lambda load-shared-object*) + (only (std native-loader) + native-loader-privileged? + native-loader-development-enabled? + native-loader-validate-library! + native-loader-validate-directory!) + (only (chezscheme) + file-regular? foreign-entry? + make-mutex mutex-acquire mutex-release)) ;; Constants are PCRE2 public ABI values. Keeping them as literals avoids ;; import-time native calls in static binaries and feature probes. @@ -98,35 +107,47 @@ (def *native-loaded?* #f) (def *bindings-ready?* #f) + (def *native-lock* (make-mutex)) + + (def (call-with-mutex mutex proc) + (dynamic-wind + (lambda () (mutex-acquire mutex)) + proc + (lambda () (mutex-release mutex)))) (def (try-load-one! path) (guard (e [(condition? e) #f]) (load-shared-object* path) #t)) - (def (try-load-any! paths) - (let loop ([paths paths]) - (cond - [(null? paths) #f] - [(try-load-one! (car paths)) #t] - [else (loop (cdr paths))]))) - - (def (native-search-paths) - (let ([base (or (getenv "JERBOA_PCRE2_LIB") ".")]) - (list - (string-append base "/jerboa_pcre2_shim.so") - (string-append base "/jerboa_pcre2_shim.dylib") - "./jerboa_pcre2_shim.so" - "./jerboa_pcre2_shim.dylib" - "jerboa_pcre2_shim.so" - "jerboa_pcre2_shim.dylib"))) + (def (configured-native-libraries) + (let ([base (getenv "JERBOA_PCRE2_LIB")]) + (unless (native-loader-development-enabled? + "JERBOA_PCRE2_DEV_NATIVE") + (error 'jerboa-pcre2-loader + "dynamic native loading requires explicit JERBOA_PCRE2_DEV_NATIVE=1")) + (native-loader-validate-directory! 'jerboa-pcre2-loader base #t) + (let ([so (string-append base "/jerboa_pcre2_shim.so")] + [dylib (string-append base "/jerboa_pcre2_shim.dylib")]) + (map (lambda (path) + (native-loader-validate-library! + 'jerboa-pcre2-loader path #t)) + (filter file-regular? (list so dylib)))))) (def (try-load-native!) (or *native-loaded?* - (and (try-load-any! (native-search-paths)) - (begin - (set! *native-loaded?* #t) - #t)))) + ;; Statically linked hosts register native symbols directly. + (and (foreign-entry? "jerboa_pcre2_compile") + (begin (set! *native-loaded?* #t) #t)) + ;; Privileged processes never consult inherited development paths. + (and (not (native-loader-privileged?)) + (let loop ([paths (configured-native-libraries)]) + (cond + [(null? paths) #f] + [(try-load-one! (car paths)) + (set! *native-loaded?* #t) + #t] + [else (loop (cdr paths))]))))) ;; ---- Native function cells ---- @@ -144,7 +165,8 @@ (def c-get-startchar #f) (def c-code-free #f) (def c-match-data-free #f) - (def c-do-substitute #f) + (def c-substitute-create #f) + (def c-substitute-code #f) (def c-substitute-result #f) (def c-substitute-result-length #f) (def c-substitute-free #f) @@ -158,8 +180,8 @@ (def c-jit-match #f) (def c-set-match-limits #f) - (def (ensure-bindings!) - (when (and (try-load-native!) (not *bindings-ready?*)) + (def (initialize-bindings!) + (when (try-load-native!) (set! c-compile (foreign-procedure __collect_safe "jerboa_pcre2_compile" (u8* size_t unsigned-32) void*)) @@ -190,15 +212,17 @@ (c-lambda (void*) int "jerboa_pcre2_code_free")) (set! c-match-data-free (c-lambda (void*) int "jerboa_pcre2_match_data_free")) - (set! c-do-substitute - (foreign-procedure __collect_safe "jerboa_pcre2_do_substitute" - (void* u8* size_t size_t unsigned-32 void* u8* size_t) integer-32)) + (set! c-substitute-create + (foreign-procedure __collect_safe "jerboa_pcre2_substitute_create" + (void* u8* size_t size_t unsigned-32 void* u8* size_t) void*)) + (set! c-substitute-code + (c-lambda (void*) integer-32 "jerboa_pcre2_substitute_code")) (set! c-substitute-result - (c-lambda () string "jerboa_pcre2_substitute_result")) + (c-lambda (void*) string "jerboa_pcre2_substitute_result")) (set! c-substitute-result-length - (c-lambda () size_t "jerboa_pcre2_substitute_result_length")) + (c-lambda (void*) size_t "jerboa_pcre2_substitute_result_length")) (set! c-substitute-free - (c-lambda () int "jerboa_pcre2_substitute_free")) + (c-lambda (void*) int "jerboa_pcre2_substitute_free")) (set! c-substring-number-from-name (c-lambda (void* string) integer-32 "jerboa_pcre2_substring_number_from_name")) (set! c-capture-count @@ -221,6 +245,12 @@ (set! *bindings-ready?* #t)) *bindings-ready?*) + (def (ensure-bindings!) + (or *bindings-ready?* + (call-with-mutex *native-lock* + (lambda () + (or *bindings-ready?* (initialize-bindings!)))))) + (def (need-native who) (unless (ensure-bindings!) (error who "unable to load jerboa_pcre2_shim native library")) @@ -346,28 +376,32 @@ (c-match-data-free md)) (void)) - (def (ffi-pcre2-do-substitute code subject subject-length startoffset options match-data replacement replacement-length) - (need-native 'ffi-pcre2-do-substitute) - (c-do-substitute (ensure-code 'ffi-pcre2-do-substitute code) + (def (ffi-pcre2-substitute-create code subject subject-length startoffset options match-data replacement replacement-length) + (need-native 'ffi-pcre2-substitute-create) + (c-substitute-create (ensure-code 'ffi-pcre2-substitute-create code) subject - (ensure-bytevector-len 'ffi-pcre2-do-substitute 'subject subject subject-length) - (ensure-size 'ffi-pcre2-do-substitute 'startoffset startoffset) - (ensure-u32 'ffi-pcre2-do-substitute 'options options) - (ensure-match-data 'ffi-pcre2-do-substitute match-data) + (ensure-bytevector-len 'ffi-pcre2-substitute-create 'subject subject subject-length) + (ensure-size 'ffi-pcre2-substitute-create 'startoffset startoffset) + (ensure-u32 'ffi-pcre2-substitute-create 'options options) + (ensure-match-data 'ffi-pcre2-substitute-create match-data) replacement - (ensure-bytevector-len 'ffi-pcre2-do-substitute 'replacement replacement replacement-length))) + (ensure-bytevector-len 'ffi-pcre2-substitute-create 'replacement replacement replacement-length))) + + (def (ffi-pcre2-substitute-code result) + (need-native 'ffi-pcre2-substitute-code) + (c-substitute-code (ensure-match-data 'ffi-pcre2-substitute-code result))) - (def (ffi-pcre2-substitute-result) + (def (ffi-pcre2-substitute-result result) (need-native 'ffi-pcre2-substitute-result) - (c-substitute-result)) + (c-substitute-result (ensure-match-data 'ffi-pcre2-substitute-result result))) - (def (ffi-pcre2-substitute-result-length) + (def (ffi-pcre2-substitute-result-length result) (need-native 'ffi-pcre2-substitute-result-length) - (c-substitute-result-length)) + (c-substitute-result-length (ensure-match-data 'ffi-pcre2-substitute-result-length result))) - (def (ffi-pcre2-substitute-free) + (def (ffi-pcre2-substitute-free result) (need-native 'ffi-pcre2-substitute-free) - (c-substitute-free) + (c-substitute-free (ensure-match-data 'ffi-pcre2-substitute-free result)) (void)) (def (ffi-pcre2-substring-number-from-name code name) --- a/src/jerboa-pcre2/pcre2.ss +++ b/src/jerboa-pcre2/pcre2.ss @@ -49,7 +49,8 @@ PCRE2_UNGREEDY PCRE2_LITERAL) (import (jerboa prelude) - (jerboa-pcre2 ffi)) + (jerboa-pcre2 ffi) + (only (chezscheme) make-mutex mutex-acquire mutex-release)) ;; ----------------------------------------------------------------------- ;; Record types @@ -57,11 +58,11 @@ (define-record-type pcre-regex (fields (mutable code) ; void* — pcre2_code pointer (zeroed after release) - (mutable match-data) ; void* — pcre2_match_data pointer (zeroed after release) pattern ; string — original pattern capture-count ; integer — number of capturing groups jit? ; boolean — JIT compilation succeeded - name-table) ; alist ((name . group-number) ...) + name-table ; alist ((name . group-number) ...) + lock) ; guards code lifetime against native calls (nongenerative pcre-regex)) (define-record-type pcre-match @@ -84,10 +85,7 @@ (let loop () (let ([rx (pcre2-guardian)]) (when rx - (let ([md (pcre-regex-match-data rx)] - [code (pcre-regex-code rx)]) - (when (not (zero? md)) (ffi-pcre2-match-data-free md)) - (when (not (zero? code)) (ffi-pcre2-code-free code))) + (release-regex-native! rx) (loop))))) ;; ----------------------------------------------------------------------- @@ -99,6 +97,12 @@ (def (ptr-null? ptr) (or (not ptr) (zero? ptr))) + (def (call-with-mutex mutex proc) + (dynamic-wind + (lambda () (mutex-acquire mutex)) + proc + (lambda () (mutex-release mutex)))) + (def (ensure-u32 who name n) (unless (and (integer? n) (>= n 0) (<= n max-u32)) (error who "invalid unsigned-32 value" name n)) @@ -117,8 +121,8 @@ (def (pcre-regex-live? rx) (and (pcre-regex? rx) - (not (ptr-null? (pcre-regex-code rx))) - (not (ptr-null? (pcre-regex-match-data rx))))) + (call-with-mutex (pcre-regex-lock rx) + (lambda () (not (ptr-null? (pcre-regex-code rx))))))) (def (ensure-live-regex who rx) (unless (pcre-regex? rx) @@ -127,21 +131,34 @@ (error who "pcre-regex has been released")) rx) + ;; Hold the same guard from the live-state check through the last native use. + ;; pcre2-release! cannot free code while PROC is using it, and a release that + ;; wins the lock first makes the operation fail before entering C. + (def (with-live-regex who rx proc) + (unless (pcre-regex? rx) + (error who "expected pcre-regex" rx)) + (call-with-mutex (pcre-regex-lock rx) + (lambda () + (let ([code (pcre-regex-code rx)]) + (when (ptr-null? code) + (error who "pcre-regex has been released")) + (proc code))))) + + (def (release-regex-native! rx) + (call-with-mutex (pcre-regex-lock rx) + (lambda () + (let ([code (pcre-regex-code rx)]) + (unless (ptr-null? code) + ;; Mark closed before calling the destructor. Repeated explicit + ;; release and later guardian cleanup then become harmless. + (pcre-regex-code-set! rx 0) + (ffi-pcre2-code-free code)))))) + (def (ensure-match-data who md) (when (ptr-null? md) (error who "PCRE2 match-data allocation failed")) md) - (def (with-match-data who rx proc) - (let ([md (ensure-match-data who - (ffi-pcre2-match-data-create-from-pattern (pcre-regex-code rx)))]) - (guard (e [(condition? e) - (ffi-pcre2-match-data-free md) - (raise e)]) - (let ([result (proc md)]) - (ffi-pcre2-match-data-free md) - result)))) - (def (pcre2-set-match-limits! match-limit depth-limit) (let ([rc (ffi-pcre2-set-match-limits (ensure-u32 'pcre2-set-match-limits! 'match-limit match-limit) @@ -212,13 +229,10 @@ (if jit? (zero? (ffi-pcre2-jit-compile code PCRE2_JIT_COMPLETE)) #f)]) - (let* ([md (ffi-pcre2-match-data-create-from-pattern code)] - [cnt (ffi-pcre2-capture-count code)] + (let* ([cnt (ffi-pcre2-capture-count code)] [names (pcre2-build-name-table code)] - [rx (make-pcre-regex code md pattern cnt jit-ok? names)]) - (when (ptr-null? md) - (ffi-pcre2-code-free code) - (error 'pcre2-compile "PCRE2 match-data allocation failed")) + [rx (make-pcre-regex code pattern cnt jit-ok? names + (make-mutex))]) (register-pcre2-finalizer rx))))])) (def (pcre2-regex-impl pattern caseless multiline dotall extended @@ -280,24 +294,34 @@ (def *cache-max* 64) (def *cache* '()) + (def *cache-lock* (make-mutex)) (def (pcre2-compile/cached pattern) - (let ([entry (assoc pattern *cache*)]) - (if entry - (let ([rx (cdr entry)]) - (if (pcre-regex-live? rx) - (begin - ;; Move to front - (set! *cache* (cons entry (remq entry *cache*))) - rx) - (begin - (set! *cache* (remq entry *cache*)) - (pcre2-compile/cached pattern)))) - (let ([rx (pcre2-compile pattern)]) - (set! *cache* (cons (cons pattern rx) *cache*)) - (when (> (length *cache*) *cache-max*) - (set! *cache* (list-head *cache* *cache-max*))) - rx)))) + (call-with-mutex *cache-lock* + (lambda () + (let ([entry (assoc pattern *cache*)]) + (if (and entry (pcre-regex-live? (cdr entry))) + (begin + ;; Move to front while still holding the cache guard. + (set! *cache* (cons entry (remq entry *cache*))) + (cdr entry)) + (begin + (when entry + (set! *cache* (remq entry *cache*))) + ;; Compile under the cache guard so two misses cannot publish + ;; duplicate entries or race the fixed-size LRU bound. + (let ([rx (pcre2-compile pattern)]) + (set! *cache* (cons (cons pattern rx) *cache*)) + (when (> (length *cache*) *cache-max*) + (set! *cache* (list-head *cache* *cache-max*))) + rx))))))) + + (def (remove-from-cache! regex) + (call-with-mutex *cache-lock* + (lambda () + (set! *cache* + (filter (lambda (entry) (not (eq? (cdr entry) regex))) + *cache*))))) (def (ensure-regex pattern-or-regex) (cond @@ -312,45 +336,41 @@ (def pcre2-do-match (case-lambda [(rx subject start options) - (pcre2-do-match rx subject start options #f #f)] - [(rx subject start options md) - (pcre2-do-match rx subject start options md #f)] - [(rx subject start options md subject-bytes) - (let* ([rx (ensure-live-regex 'pcre2-do-match rx)] - [subject (ensure-string 'pcre2-do-match 'subject subject)] + (pcre2-do-match rx subject start options #f)] + [(rx subject start options subject-bytes) + (let* ([subject (ensure-string 'pcre2-do-match 'subject subject)] [start (ensure-start-index 'pcre2-do-match subject start)] [options (ensure-u32 'pcre2-do-match 'options options)] - [code (pcre-regex-code rx)] - [own-md? (not md)] - [md (ensure-match-data 'pcre2-do-match - (if md md (ffi-pcre2-match-data-create-from-pattern code)))] [bv (if subject-bytes subject-bytes (string->utf8 subject))] [byte-start (if (zero? start) 0 (char-index->byte-offset bv start))]) - (guard (e [(condition? e) - (when own-md? (ffi-pcre2-match-data-free md)) - (raise e)]) - (let ([rc (if (pcre-regex-jit? rx) - (ffi-pcre2-jit-match code bv (bytevector-length bv) - byte-start options md) - (ffi-pcre2-match code bv (bytevector-length bv) - byte-start options md))]) - (if (< rc 0) - (begin - (when own-md? (ffi-pcre2-match-data-free md)) - #f) - (let* ([ncap (+ (pcre-regex-capture-count rx) 1)] - [sv (make-vector ncap #f)]) - (let loop ([i 0]) - (when (< i ncap) - (unless (ffi-pcre2-ovector-is-unset? md i) - (let ([bstart (ffi-pcre2-ovector-start md i)] - [bend (ffi-pcre2-ovector-end md i)]) - (vector-set! sv i - (cons (byte-offset->char-index bv bstart) - (byte-offset->char-index bv bend))))) - (loop (+ i 1)))) - (when own-md? (ffi-pcre2-match-data-free md)) - (make-pcre-match sv subject (pcre-regex-name-table rx)))))))])) + (with-live-regex 'pcre2-do-match rx + (lambda (code) + (let ([md (ensure-match-data 'pcre2-do-match + (ffi-pcre2-match-data-create-from-pattern code))]) + (dynamic-wind + (lambda () (void)) + (lambda () + (let ([rc (if (pcre-regex-jit? rx) + (ffi-pcre2-jit-match code bv (bytevector-length bv) + byte-start options md) + (ffi-pcre2-match code bv (bytevector-length bv) + byte-start options md))]) + (if (< rc 0) + #f + (let* ([ncap (+ (pcre-regex-capture-count rx) 1)] + [sv (make-vector ncap #f)]) + (let loop ([i 0]) + (when (< i ncap) + (unless (ffi-pcre2-ovector-is-unset? md i) + (let ([bstart (ffi-pcre2-ovector-start md i)] + [bend (ffi-pcre2-ovector-end md i)]) + (vector-set! sv i + (cons (byte-offset->char-index bv bstart) + (byte-offset->char-index bv bend))))) + (loop (+ i 1)))) + (make-pcre-match sv subject + (pcre-regex-name-table rx)))))) + (lambda () (ffi-pcre2-match-data-free md)))))))])) ;; ----------------------------------------------------------------------- ;; Matching API @@ -375,24 +395,8 @@ (case-lambda [(rx/str subject) (pcre2-matches? rx/str subject 0)] [(rx/str subject start) - (let* ([subject (ensure-string 'pcre2-matches? 'subject subject)] - [start (ensure-start-index 'pcre2-matches? subject start)] - [rx (ensure-regex rx/str)] - [code (pcre-regex-code rx)] - [md (ensure-match-data 'pcre2-matches? - (ffi-pcre2-match-data-create 1))] - [bv (string->utf8 subject)] - [byte-start (if (zero? start) 0 (char-index->byte-offset bv start))]) - (guard (e [(condition? e) - (ffi-pcre2-match-data-free md) - (raise e)]) - (let ([rc (if (pcre-regex-jit? rx) - (ffi-pcre2-jit-match code bv (bytevector-length bv) - byte-start 0 md) - (ffi-pcre2-match code bv (bytevector-length bv) - byte-start 0 md))]) - (ffi-pcre2-match-data-free md) - (>= rc 0))))])) + (let ([rx (ensure-regex rx/str)]) + (and (pcre2-do-match rx subject start 0) #t))])) ;; ----------------------------------------------------------------------- ;; Match result access @@ -442,6 +446,39 @@ ;; Substitution ;; ----------------------------------------------------------------------- + (def (pcre2-substitute who rx subject replacement start options) + (let* ([bv-subj (string->utf8 subject)] + [bv-repl (string->utf8 replacement)] + [byte-start (if (zero? start) 0 + (char-index->byte-offset bv-subj start))]) + (with-live-regex who rx + (lambda (code) + ;; Both match data and the result object are operation-owned. No + ;; mutable PCRE2 buffer is shared by concurrent callers. + (let ([md (ensure-match-data who + (ffi-pcre2-match-data-create-from-pattern code))]) + (dynamic-wind + (lambda () (void)) + (lambda () + (let ([result + (ffi-pcre2-substitute-create + code bv-subj (bytevector-length bv-subj) + byte-start options md + bv-repl (bytevector-length bv-repl))]) + (when (ptr-null? result) + (error who "PCRE2 substitution allocation failed")) + (dynamic-wind + (lambda () (void)) + (lambda () + (let ([rc (ffi-pcre2-substitute-code result)]) + (cond + [(>= rc 0) (ffi-pcre2-substitute-result result)] + [(= rc PCRE2_ERROR_NOMATCH) subject] + [else (error who + (ffi-pcre2-get-error-message rc) rc)]))) + (lambda () (ffi-pcre2-substitute-free result))))) + (lambda () (ffi-pcre2-match-data-free md)))))))) + (def pcre2-replace (case-lambda [(rx/str subject replacement) @@ -453,25 +490,8 @@ [replacement (ensure-string 'pcre2-replace 'replacement replacement)] [start (ensure-start-index 'pcre2-replace subject start)] [rx (ensure-regex rx/str)] - [opts (if extended? PCRE2_SUBSTITUTE_EXTENDED 0)] - [bv-subj (string->utf8 subject)] - [bv-repl (string->utf8 replacement)] - [byte-start (if (zero? start) 0 - (char-index->byte-offset bv-subj start))] - [rc (ffi-pcre2-do-substitute - (pcre-regex-code rx) - bv-subj (bytevector-length bv-subj) - byte-start opts - (pcre-regex-match-data rx)