add fixes
ober
0ad30ecb54fd705feb6ca739c3b5afb7e54bbac7
--- a/data/changelog.sexp +++ b/data/changelog.sexp @@ -2,7 +2,26 @@ . "Machine-readable changelog of Jerboa API drift. Consumers (LLM tooling, lints, jerboa_verify) use this to invalidate stale recommendations and to suggest migrations when a symbol is renamed or relocated.") ("entries" - (("added") + (("added" "sqlite-open" "sqlite-close" "sqlite-exec" + "sqlite-execute" "sqlite-query" "sqlite-prepare" + "sqlite-finalize" "sqlite-step" "sqlite-bind" + "tcp-connect" "tcp-listen" "tcp-accept" "tcp-close" + "tcp-read" "tcp-write" "tcp-write-string" + "open-safe-input-file" "open-safe-output-file" + "call-with-safe-input-file" "call-with-safe-output-file" + "*safe-mode*") + ("date" . "2026-06-07") + ("modules_added") + ("moved") + ("notes" + . + "MCP tooling now ranks feature suggestions by votes/impact, ranks cookbook search by tokenized relevance across id/title/tags/notes/code, and expands jerboa_boot_library_audit into a transitive std/jerboa import closure audit with roots/entry_paths and external_libs diff reporting. jerboa_verify and jerboa_compile_check now synthesize a jerbuild-style library wrapper for .ss source files with top-level export forms, including Jerboa auto-imports and filtered Chez exclusions. The translator string scanners now stop safely when escaped-string traversal passes the target index and skip #\\ character literals before bracket/method-dispatch rewriting, fixing reproduced large-file string-ref and #\\{/#\\[ verification failures. (jerboa prelude) now exports safe sqlite/TCP/file helpers under the simple names from (std safe), so sqlite-open, tcp-connect, and related APIs default to the contract-checked wrappers. jerboa_doc and jerboa_module_catalog now include usage examples from curated calling-convention overrides, cookbook snippets, or arity-aware fallback skeletons; the metrics make-counter example shows registry-first usage.") + ("removed") + ("renamed") + ("tier_changes") + ("tools_added") + ("version" . #f)) + (("added") ("date" . "2026-06-03") ("modules_added") ("moved") --- a/data/cookbooks.sexp +++ b/data/cookbooks.sexp @@ -5303,4 +5303,63 @@ "liveness" "busy-state") ("title" . - "Track spawned worker liveness with thread-done?"))) + "Track spawned worker liveness with thread-done?")) + (("code" + . + ";; Pattern for wrapping a C-shaped cdylib API from Jerboa/Chez.\n;; The native side should expose demo_new/demo_free/demo_row_text/demo_last_error.\n\n(define shlib-ext\n (let ([mt (symbol->string (machine-type))])\n (if (and (>= (string-length mt) 3)\n (string=? (substring mt (- (string-length mt) 3) (string-length mt)) \"osx\"))\n \"dylib\"\n \"so\")))\n\n(define static-build?\n (let ([v (getenv \"JEMACS_STATIC\")])\n (and v (not (string=? v \"\")) (not (string=? v \"0\")))))\n\n(define demo-lib-path\n (or (getenv \"DEMO_LIB\")\n (string-append (or (getenv \"DEMO_DIR\") \".\") \"/libdemo.\" shlib-ext)))\n\n(define demo-lib-loaded\n (if static-build? #f (load-shared-object demo-lib-path)))\n\n(define ffi-demo-new (foreign-procedure \"demo_new\" () void*))\n(define ffi-demo-free (foreign-procedure \"demo_free\" (void*) void))\n(define ffi-demo-row-text (foreign-procedure \"demo_row_text\" (void* int u8* size_t) int))\n(define ffi-demo-last-error (foreign-procedure \"demo_last_error\" (u8* size_t) int))\n\n(define (demo-last-error)\n (let* ([buf (make-bytevector 4096 0)]\n [n (ffi-demo-last-error buf 4096)])\n (if (> n 0)\n (utf8->string (let ([out (make-bytevector n)])\n (bytevector-copy! buf 0 out 0 n)\n out))\n \"unknown native error\")))\n\n(define (check-rc who rc)\n (if (< rc 0) (error who (demo-last-error)) rc))\n\n(define (make-demo-session)\n (let ([handle (ffi-demo-new)])\n (if handle (box handle) (error 'make-demo-session (demo-last-error)))))\n\n(define (demo-session-handle! session)\n (let ([handle (unbox session)])\n (if handle handle (error 'demo-session \"native session is closed\"))))\n\n(define (demo-session-free! session)\n (let ([handle (unbox session)])\n (when handle\n (ffi-demo-free handle)\n (set-box! session #f))))\n\n(define (demo-row-text session row)\n (let* ([handle (demo-session-handle! session)]\n [buf (make-bytevector 16384 0)]\n [n (ffi-demo-row-text handle row buf 16384)])\n (check-rc 'demo-row-text n)\n (if (> n 0)\n (utf8->string (let ([out (make-bytevector n)])\n (bytevector-copy! buf 0 out 0 n)\n out))\n \"\")))\n\n;; Use dynamic-wind at call sites when possible so native handles are freed.\n;; (let ([s (make-demo-session)])\n;; (dynamic-wind void (lambda () (demo-row-text s 0))\n;; (lambda () (demo-session-free! s))))") ("id" . "chez-cdylib-opaque-handle-wrapper") + ("imports" "(chezscheme)") + ("notes" + . + "Keep the native ABI C-shaped: opaque pointer lifecycle, integer return codes, explicit last-error retrieval, and caller-owned output buffers. Load the shared object unless running a static build where symbols are already linked. After free, set the handle box to #f so double-free and use-after-free become Scheme errors. For output buffers declared as `u8*`, pass Scheme bytevectors; use `void*` when passing foreign-allocated pointers instead.") + ("tags" "ffi" "foreign-procedure" "cdylib" "opaque-handle" + "bytevector" "dynamic-library") + ("title" + . + "Wrap a Rust/C cdylib opaque handle from Jerboa Scheme")) + (("code" + . + "# Makefile pattern for consuming a sibling Jerboa project that provides\n# generated Scheme modules plus a native cdylib used by those modules.\n\nJERBOA_TERM_ROOT ?= $(abspath ../jerboa-term)\nJERBOA_TERM_LIBDIR ?= $(JERBOA_TERM_ROOT)/lib\n\n# Add the sibling generated library directory to --libdirs so imports like\n# (import (jerboa-term alacritty-term)) or :jerboa-term/alacritty-term resolve.\nLIBDIRS = --libdirs lib:$(JERBOA)/lib:$(JERBOA_TERM_LIBDIR)\n\n# Export a runtime path used by the sibling Scheme wrapper to load its cdylib.\n# The wrapper can also honor an exact JERBOA_TERM_LIB override.\nexport JERBOA_TERM_DIR ?= $(JERBOA_TERM_ROOT)/target/debug\n\n# Build the sibling native library before running tests that import it.\ntest-alacritty-term: build\n\tcd $(JERBOA_TERM_ROOT) && cargo build --lib >/dev/null\n\t$(SCHEME) $(LIBDIRS) --script tests/test-alacritty-term.ss\n\n# In the sibling Scheme wrapper:\n# (define lib-path\n# (or (getenv \"JERBOA_TERM_LIB\")\n# (string-append (or (getenv \"JERBOA_TERM_DIR\") \"../jerboa-term/target/debug\")\n# \"/libjerboa_term_core.\" shlib-ext)))\n# (load-shared-object lib-path)") ("id" . "jerbuild-sibling-libdir-shlib-env") ("imports") + ("notes" + . + "There are two separate paths to wire: the Jerboa module search path (`--libdirs`) and the native dynamic-library path (`JERBOA_TERM_DIR` or an exact `JERBOA_TERM_LIB`). Building the sibling Rust/C library inside the test target prevents stale symbols when the Scheme FFI wrapper was regenerated but the cdylib was not.") + ("tags" "jerbuild" "libdirs" "sibling-project" + "shared-library" "makefile" "ffi") + ("title" + . + "Import a sibling Jerboa project and load its native shared library")) + (("code" + . + "(import (jerboa prelude))\n\n(def (estimate-history-tokens messages)\n ;; Simple estimator for message history only. Real chat requests also spend\n ;; tokens on system prompt, JSON envelope, and tool schemas.\n (quotient (apply + (map string-length messages)) 4))\n\n(def (effective-history-budget context-window)\n ;; For tiny local tool-calling models, reserve most of the advertised\n ;; working window for fixed request overhead.\n (cond\n ((and context-window (<= context-window 4096))\n (max 512 (- context-window 2500)))\n (else context-window)))\n\n(def (should-compact? messages context-window trigger-pct)\n (let* ((budget (effective-history-budget context-window))\n (est (estimate-history-tokens messages)))\n (and budget (> est 0)\n (>= (* 100 est) (* trigger-pct budget)))))\n\n(displayln (should-compact?\n (list (make-string 1600 #\\s) \"find bugs\" (make-string 768 #\\r))\n 3072\n 80))") ("id" . "reserve-local-prompt-overhead-for-compaction") + ("imports" "(jerboa prelude)") + ("notes" + . + "Use this when the estimator only sees message history. Local tool-calling requests can have thousands of fixed tokens from the system prompt and tool schemas, so compaction that compares history directly to the full model window fires too late. In jcode this avoided MLX prefill stalls where prompt_tokens climbed from ~2500 to ~9000 even though the message-only estimate looked acceptable.") + ("tags" "compaction" "local-model" "tool-schema" + "context-window" "prompt-overhead" "jcode") + ("title" + . + "Reserve fixed prompt overhead when compacting local tool-calling chats")) + (("code" + . + "(import (jerboa prelude))\n\n;; Messages are represented as (role . text) for this standalone example.\n(def (message-role m) (car m))\n\n(def (drop-middle-keep-recent messages keep-recent)\n \"Keep system + first user + the last KEEP-RECENT iterations.\"\n (let* ((head (take messages 2))\n (tail-count (min (length messages) (* keep-recent 2)))\n (tail (drop messages (max 2 (- (length messages) tail-count)))))\n (append head tail)))\n\n(def (compact-for-budget messages budget)\n (cond\n ;; Tiny budgets cannot afford old tool-call skeletons piling up. Drop\n ;; middle history completely, but keep enough recent context to avoid\n ;; immediate tool repetition.\n ((and budget (<= budget 1024))\n (drop-middle-keep-recent messages 2))\n (else\n messages)))\n\n(def sample\n (list (cons \"system\" \"s\")\n (cons \"user\" \"q\")\n (cons \"assistant\" \"read README\")\n (cons \"tool\" \"README contents\")\n (cons \"assistant\" \"repomap\")\n (cons \"tool\" \"top files\")\n (cons \"assistant\" \"read Makefile\")\n (cons \"tool\" \"Makefile contents\")))\n\n(displayln (length (compact-for-budget sample 572)))") ("id" . "sliding-compaction-for-tiny-contexts") + ("imports" "(jerboa prelude)") + ("notes" + . + "Tiered compaction that preserves old tool-call skeletons can still let tiny local prompts grow round by round. For very small effective budgets, use sliding-window compaction to drop older iterations entirely. Keeping one iteration was too aggressive in jcode and caused repeated read/repomap calls; keeping two recent iterations balanced context retention against prompt size. Pair this with smaller tool-result previews.") + ("tags" "compaction" "sliding-window" "tiny-context" + "keep-recent" "tool-results" "jcode") + ("title" + . + "Use sliding-window compaction for tiny context budgets")) + (("code" + . + "(import (jerboa prelude))\n\n(def (refresh-context messages)\n ;; Replace the leading system message and run any compaction policy.\n ;; Real code would rebuild the system prompt and compact old history here.\n (let ((fresh-system \"system: fresh mode/tool/context instructions\"))\n (cond\n ((null? messages) (list fresh-system))\n (else (cons fresh-system (cdr messages))))))\n\n(def (fake-model messages)\n ;; A fake tool-using model: call one tool, then stop.\n (if (< (length messages) 4) 'tool-call 'done))\n\n(def (execute-tool _call)\n \"tool result\")\n\n(def (chat-loop messages round)\n ;; Important: refresh at the start of every recursive round, not only when\n ;; the conversation is first created.\n (let* ((msgs (refresh-context messages))\n (reply (fake-model msgs)))\n (cond\n ((eq? reply 'done) msgs)\n (else\n (chat-loop\n (append msgs (list \"assistant: tool-call\" (execute-tool reply)))\n (+ round 1))))))\n\n(displayln (length (chat-loop (list \"old system\" \"user: inspect repo\") 0)))") ("id" . "refresh-chat-context-each-round") + ("imports" "(jerboa prelude)") + ("notes" + . + "If refresh/compaction is only applied in one caller path, other paths such as non-session CLI loops can keep appending unbounded history. In jcode, the TUI/session path refreshed but plain agent-chat-loop and tracking loops did not, so local MLX prompts kept growing despite compaction code existing. Put refresh at the top of every recursive model-call loop and append tool results to the refreshed message list.") + ("tags" "chat-loop" "compaction" "system-prompt" + "recursive-loop" "context-refresh" "jcode") + ("title" + . + "Refresh chat context before every model round"))) --- a/data/features.sexp +++ b/data/features.sexp @@ -56,10 +56,15 @@ "Claude writes (import (jerboa prelude)) then (sqlite-open \"db\") — this silently calls safe-sqlite-open which validates types, and (with-resource ...) is the documented pattern. No separate (import (std safe)) needed.") ("id" . "jerboa-safe-prelude-default") ("impact" . "critical") + ("implemented_in" . "lib/jerboa/prelude.ss") + ("implemented_tool" . "(jerboa prelude)") ("note" . - "Requires Jerboa core change (prelude.sls), not implementable in jerboa-mcp.") - ("status" . "core-backlog") + "Implemented in the core prelude by importing (std safe) with a private prefix and exporting safe sqlite/TCP/file aliases under the simple names.") + ("status" . "implemented") + ("closed_reason" + . + "(jerboa prelude) now exports sqlite-open/sqlite-close/sqlite-exec/sqlite-query/sqlite-prepare/sqlite-finalize/sqlite-step/sqlite-bind, tcp-connect/tcp-listen/tcp-accept/tcp-close/tcp-read/tcp-write/tcp-write-string, open-safe-input-file/open-safe-output-file, call-with-safe-input-file/call-with-safe-output-file, and *safe-mode* as aliases to (std safe). Raw APIs remain opt-in through explicit native module imports.") ("tags" "safety" "prelude" "default" "claude" "api" "import") ("title" . "Make safe APIs the default prelude exports") @@ -231,6 +236,12 @@ "Implementing (std metrics) in jerboa-edge Phase 3: make-counter reported \"3 or more arguments (variadic, arity mask: -8)\" — unhelpful. Three successive jerboa_eval attempts failed (registry last, then registry as #f, then keyword syntax) before reading the full 200-line stdlib source to discover that registry is the FIRST argument: (make-counter reg name help [label-names]). A jerboa_doc call returning \"(make-counter reg name help) ;; reg is the registry\" would have resolved this in 1 tool call instead of 5 (3 eval + 1 stdlib_source read + 1 eval to confirm).") ("id" . "module-doc-with-calling-convention-example") ("impact" . "high") + ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_doc, jerboa_module_catalog") + ("status" . "implemented") + ("closed_reason" + . + "jerboa_doc now appends an Example line for looked-up symbols, and jerboa_module_catalog includes an Example column for every export. Examples come from curated calling-convention overrides for known tricky APIs, existing cookbook snippets when available, and arity-aware fallback call skeletons for the rest.") ("tags" "doc" "example" "calling-convention" "module-catalog" "arity" "usage") ("title" @@ -486,9 +497,12 @@ . "Static-check any .ss file that imports another module from the same project before runtime. Currently any nontrivial project file fails compile_check, forcing fallback to either jerboa_make (which only reports linker errors), the integration test (which is slow), or jerboa_eval (which doesn't catch unbound-identifier errors at expand time).") ("votes" . 3)) - (("description" + (("closed_reason" . - "When building a static jerboa binary via make-boot-file + compile-program, every transitive (std ...) and (jerboa ...) library import of every included module must be explicitly listed in the binary's `external-libs` list. If even one is missing, the build succeeds but the binary crashes at runtime with `Exception: library (std X Y) not found`. There is no compile-time check for this — each missing library surfaces as a separate runtime failure, requiring a full Docker rebuild cycle (~5 minutes) to discover. A tool that takes a set of root .ss/.sls files (or a project's existing external-libs list) and computes the actual transitive closure of stdlib imports would catch all gaps in one shot. Implementation: parse the import forms in each .ss/.sls under a given libdir set, follow the graph until fixed point, return the diff against the existing external-libs list.") + "jerboa_boot_library_audit now accepts roots/entry_paths plus external_libs, resolves imports through project/Jerboa libdirs, follows transitive std/jerboa imports, and reports missing/extra external libraries.") + ("description" + . + "When building a static jerboa binary via make-boot-file + compile-program, every transitive (std ...) and (jerboa ...) library import of every included module must be explicitly listed in the binary's `external-libs` list. If even one is missing, the build succeeds but the binary crashes at runtime with `Exception: library (std X Y) not found`. There is no compile-time check for this — each missing library surfaces as a separate runtime failure, requiring a full Docker rebuild cycle (~5 minutes) to discover. A tool that takes a set of root .ss/.sls files (or a project's existing external-libs list) and computes the actual transitive closure of stdlib imports would catch all gaps in one shot. Implementation: parse the import forms in each .ss/.sls under a given libdir set, follow the graph until fixed point, return the diff against the existing external-libs list.") ("estimated_token_reduction" . "~2000 tokens per binary integration of new vendored library (eliminates 3-4 full build/crash/grep/fix cycles, each ~500-700 tokens)") @@ -500,8 +514,8 @@ ("implemented_tool" . "jerboa_boot_library_audit") ("note" . - "Implemented as boot-library/import closure audit; project-specific external-libs diffing can be expanded later.") - ("status" . "partial") + "Implemented as boot-library/import closure audit with multiple roots, transitive std/jerboa dependency walking, and explicit external_libs diff reporting.") + ("status" . "implemented") ("tags" "static-binary" "external-libs" "boot-file" "library-closure" "transitive-deps") ("title" @@ -531,9 +545,12 @@ . "Before building or shipping a static Jerboa binary after switching between Linux, FreeBSD, and macOS build targets, especially when vendored library roots contain cached .so/.wpo files.") ("votes" . 0)) - (("description" + (("closed_reason" . - "jerboa_verify currently treats a repo source file that begins with `(export #t)` as invalid with `export form outside of a module or library`, even though this is the expected source form for jerbuild-managed project modules such as jerboa-shell's glob.ss. The verifier should detect jerbuild-style .ss files, wrap or translate them the same way jerbuild does, and avoid unrelated divergence false positives on local variable names like `len`.") + "jerboa_verify and jerboa_compile_check detect .ss files with top-level export forms, synthesize a jerbuild-style library wrapper, add Jerboa auto-imports and Chez exclusions, and verify the wrapped module.") + ("description" + . + "jerboa_verify currently treats a repo source file that begins with `(export #t)` as invalid with `export form outside of a module or library`, even though this is the expected source form for jerbuild-managed project modules such as jerboa-shell's glob.ss. The verifier should detect jerbuild-style .ss files, wrap or translate them the same way jerbuild does, and avoid unrelated divergence false positives on local variable names like `len`.") ("estimated_token_reduction" . "~400 tokens per Jerboa source edit, plus avoids a 20-30 second full-build fallback for simple syntax checks.") @@ -541,7 +558,9 @@ . "Editing /Users/user/mine/jerboa-shell/glob.ss and running jerboa_verify reported `export form outside of a module or library (export #t)`, so the only useful verification was a full `make jsh-macos-full` build. A verifier mode that follows jerbuild source semantics would have caught syntax/compile errors without a full binary rebuild.") ("id" . "verify-jerbuild-export-source-support") - ("impact" . "medium") + ("impact" . "medium") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_verify") + ("status" . "implemented") ("tags" "verify" "jerbuild" "export" "source" "project") ("title" . @@ -1125,14 +1144,20 @@ . "When validating larger Jerboa scripts or generated files, jerboa_verify should return a structured parse/compile result instead of an internal exception.") ("votes" . 0)) - (("description" + (("closed_reason" . - "jerboa_howto search is recipe-matches? (a plain case-insensitive SUBSTRING test over id/title/code/tags -- notes are not searched) followed by (take matches max_results) in cookbook FILE ORDER (mcp/server.ss ~L680, L925). Two consequences: (1) Recipes added via jerboa_howto_add are appended LAST, so a broad single-term query truncates them below the take(max_results) cut. (2) A multi-word query is tested as one literal substring, so natural queries match nothing because no single field contains the whole phrase. Proposed fix: tokenize the query (split on whitespace), score each recipe by term hits with field weighting (id/title/tags > code, and include notes), sort by score, THEN take(max_results). Small, self-contained change to recipe-matches? and the howto search handler. Complements (does not duplicate) the broader jerboa-semantic-search-stdlib proposal, which adds a separate embeddings-based tool.") ("estimated_token_reduction" . "") + "jerboa_howto tokenizes natural-language queries, scores id/title/tags/notes/code with field weighting, sorts by relevance before limiting, and returns exact/new recipe matches ahead of older broad matches.") + ("description" + . + "jerboa_howto search is recipe-matches? (a plain case-insensitive SUBSTRING test over id/title/code/tags -- notes are not searched) followed by (take matches max_results) in cookbook FILE ORDER (mcp/server.ss ~L680, L925). Two consequences: (1) Recipes added via jerboa_howto_add are appended LAST, so a broad single-term query truncates them below the take(max_results) cut. (2) A multi-word query is tested as one literal substring, so natural queries match nothing because no single field contains the whole phrase. Proposed fix: tokenize the query (split on whitespace), score each recipe by term hits with field weighting (id/title/tags > code, and include notes), sort by score, THEN take(max_results). Small, self-contained change to recipe-matches? and the howto search handler. Complements (does not duplicate) the broader jerboa-semantic-search-stdlib proposal, which adds a separate embeddings-based tool.") + ("estimated_token_reduction" . "") ("example_scenario" . "This session: after jerboa_howto_add of recipe 'match-predicate-bind' (id, title, and tags all contain 'match'), jerboa_howto query 'match' did not surface it until max_results>=43 -- it was the 43rd of 43 substring hits, last because it was newest. The multi-word query 'match predicate bind' returned nothing (no field contains that literal phrase). jerboa_howto_get by exact id worked, so the recipe was stored fine; it was simply unrankable/unreachable via keyword search. New cookbook knowledge is effectively invisible to search until queried very specifically.") ("id" . "howto-search-relevance-ranking") - ("impact" . "high") ("status" . "proposed") + ("impact" . "high") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_howto") + ("status" . "implemented") ("tags" "howto" "cookbook" "search" "ranking" "tokenize" "mcp" "discoverability") ("title" @@ -1177,9 +1202,13 @@ . "Use when validating changed Jerboa source files before build, especially larger modules such as src/jcode/core/agent.ss or src/jcode/ui/serve.ss.") ("votes" . 0)) - (("description" + (("closed_reason" . - "jerboa_verify on a repo source file such as src/jcode/ui/tui-sidebar.ss reports \"export form outside of a module or library\" even though make build transpiles and compiles the file successfully. The verifier should load/transpile .ss source files with the same module context/libdirs used by jerbuild so local modules with top-level export forms can be checked directly.") ("estimated_token_reduction" . "") + "Project .ss files with top-level export forms are verified in a synthesized module context with project libdirs and jerbuild-style base imports.") + ("description" + . + "jerboa_verify on a repo source file such as src/jcode/ui/tui-sidebar.ss reports \"export form outside of a module or library\" even though make build transpiles and compiles the file successfully. The verifier should load/transpile .ss source files with the same module context/libdirs used by jerbuild so local modules with top-level export forms can be checked directly.") + ("estimated_token_reduction" . "") ("example_scenario" . "Run jerboa_verify with file_path=/Users/user/mine/jerboa-code/src/jcode/ui/tui-sidebar.ss and project_path=/Users/user/mine/jerboa-code. Expected: verify the module. Actual: export form outside of a module or library.") @@ -1187,7 +1216,9 @@ ("impact" . "Reduces false verification failures and avoids fallback build-only checks for ordinary Jerboa project source files.") - ("status" . "open") + ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_verify") + ("status" . "implemented") ("tags" "verify" "export" "source-file" "project-context" "jerbuild") ("title" @@ -1208,6 +1239,12 @@ "jerboa_compile_check(file_path: '.../provider/provider.ss') on a ~94KB file returned 'Exception in string-ref: 94269 is not a valid index for \"<the full 94KB source>\"'. I had to fall back to `make binary` (minutes) to validate a one-line edit, and the error echoed the entire file into context.") ("id" . "compile-check-large-file-robustness") ("impact" . "medium") + ("implemented_in" . "lib/jerboa/translator.sls") + ("implemented_tool" . "jerboa_compile_check") + ("note" + . + "The reproduced string-ref scanner failures are fixed in the translator string scanners; separate diagnostic truncation hardening can still be improved.") + ("status" . "partial") ("tags" "compile-check" "large-file" "robustness" "error-message" "token-bloat") ("title" @@ -1314,23 +1351,28 @@ ("votes" . 0)) (("description" . - "jerboa_verify and jerboa_compile_check crashed internally with `Exception in string-ref: 100111 is not a valid index` while checking src/jcode/provider/provider.ss. The normal jerbuild build compiled the file successfully, so the MCP verifier appears to have a source-indexing or pre-scan bug on large files. The tool should return a compact diagnostic with the failing pass/name and a usable location, or fall back to compile-only verification instead of dumping the entire file string.") + "jerboa_verify repeatedly crashed with an internal string-ref out-of-range exception while checking valid jcode source files such as compaction.ss and compaction-strategy.ss. The fallback was to run the full make test/build path, which works but costs substantially more time and tokens. The verifier should catch parser/indexing errors and either complete normally or return a clear diagnostic with file/line context.") ("estimated_token_reduction" . - "~2000-5000 tokens per failure by avoiding huge file dumps and fallback investigation.") + "~3k-8k tokens per affected edit by avoiding repeated verifier output dumps and full build/test fallback discussion.") ("example_scenario" . - "After adding a TCP stream watchdog to src/jcode/provider/provider.ss, both jerboa_verify and jerboa_compile_check failed before compiling with string-ref index 100111, while jerboa_make build passed.") + "After editing src/jcode/core/compaction-strategy.ss, jerboa_verify reported: Exception in string-ref: 11303 is not a valid index for the whole source string. The source later passed make test, so the verifier failure was tool-side noise.") ("id" . "verify-large-file-string-ref-crash") - ("impact" . "medium") - ("tags" "verify" "compile_check" "large-file" "diagnostics" - "source-index") + ("impact" . "high") ("implemented_in" . "lib/jerboa/translator.sls") + ("implemented_tool" . "jerboa_verify") + ("closed_reason" + . + "Fixed translator string scanners so escaped strings and character literals no longer advance past EOF or rewrite #\\[/#\\]/#\\{/#\\} while verifying large files.") + ("status" . "implemented") + ("tags" "verify" "parser" "large-file" "string-ref" + "diagnostics") ("title" . - "Report source location instead of crashing on large-file verify index errors") + "Make jerboa_verify robust on large source files") ("use_case" . - "When validating a large Jerboa source file after an edit, agents need to distinguish user code errors from verifier/tooling crashes without falling back to noisy shell builds.") + "Use jerboa_verify as the mandatory fast pre-build validation step after editing Jerboa source files, including larger modules.") ("votes" . 0)) (("description" . @@ -1350,9 +1392,12 @@ . "Every multi-line edit to .ss source in any jerboa repo — the mandated safe-edit workflow is currently impossible to follow.") ("votes" . 0)) - (("description" + (("closed_reason" . - "jerbuild source files start with a bare (export ...) + (import :pkg/path ...) that jerbuild wraps into a library at build time. jerboa_compile_check feeds the raw file to the expander and dies with 'export form outside of a module or library', so per-file compile validation is impossible in jerbuild projects like jcode — the only check is a full project build. compile_check could detect the leading export form and synthesize the library wrapper the way jerbuild does.") + "jerboa_compile_check now routes top-level export .ss files through the same synthesized jerbuild-style module wrapper used by jerboa_verify.") + ("description" + . + "jerbuild source files start with a bare (export ...) + (import :pkg/path ...) that jerbuild wraps into a library at build time. jerboa_compile_check feeds the raw file to the expander and dies with 'export form outside of a module or library', so per-file compile validation is impossible in jerbuild projects like jcode — the only check is a full project build. compile_check could detect the leading export form and synthesize the library wrapper the way jerbuild does.") ("estimated_token_reduction" . "Replaces a full build cycle (or blind commit) per edit with one tool call; ~2-5k tokens saved per validation round.") @@ -1360,7 +1405,9 @@ . "After editing src/jcode/core/config.ss, jerboa_compile_check file_path=... failed on the file's own (export load-config ...) header. Fallback was jerboa_check_syntax on a hand-pasted snippet, which validates syntax but not bindings/arity.") ("id" . "compile-check-jerbuild-modules") - ("impact" . "medium") + ("impact" . "medium") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_compile_check") + ("status" . "implemented") ("tags" "compile-check" "jerbuild" "export" "module" "validation") ("title" @@ -1370,9 +1417,12 @@ . "Validating a single edited src file in any jerbuild-managed project without paying a multi-minute full build (or in jcode's case `make binary`, since `make build` deadlocks).") ("votes" . 0)) - (("description" + (("closed_reason" . - "Allow jerboa_verify and jerboa_compile_check to handle repository source files that use the project/transpiler top-level .ss style with import/export forms, instead of reporting \"export form outside of a module or library\". The tool should resolve project_path/local libdirs and run the same wrapper/transpile context as make build where practical.") + "jerboa_verify and jerboa_compile_check now handle top-level export .ss project source files by compiling a synthesized library form.") + ("description" + . + "Allow jerboa_verify and jerboa_compile_check to handle repository source files that use the project/transpiler top-level .ss style with import/export forms, instead of reporting \"export form outside of a module or library\". The tool should resolve project_path/local libdirs and run the same wrapper/transpile context as make build where practical.") ("estimated_token_reduction" . "~800-1500 tokens per affected edit by avoiding failed verifier debugging and reducing fallback build iterations.") @@ -1380,7 +1430,9 @@ . "While editing src/jcode/ui/tui-message.ss, jerboa_verify failed with \"export form outside of a module or library\" even though make build transpiled and compiled the file successfully.") ("id" . "verify-top-level-project-ss-files") - ("impact" . "medium") + ("impact" . "medium") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_verify") + ("status" . "implemented") ("tags" "verify" "compile-check" "project-path" "top-level-ss" "exports") ("title" @@ -1426,7 +1478,9 @@ . "When checking a large Jerboa source file after a small edit, users need actionable syntax/compile diagnostics rather than a tool-internal string-ref exception.") ("votes" . 0)) - (("closed_reason" . "") + (("closed_reason" + . + "Top-level export .ss files are wrapped as synthetic libraries for both verify and compile_check, using project_path libdirs and Jerboa auto-imports.") ("description" . "jerboa_verify and jerboa_compile_check currently report \"export form outside of a module or library\" for project .ss files that start with top-level export forms, even though jerbuild transpile/build accepts the same files and compiles them into libraries. The tools should use the same project-aware wrapping/transpile path as the build for user-facing .ss files.") @@ -1437,8 +1491,9 @@ . "After editing src/jcode/core/message.ss, jerboa_verify and jerboa_compile_check failed immediately on the leading export form, while jerboa_make build successfully transpiled and compiled the file. The session had to fall back to make build/test for compile validation.") ("id" . "verify-top-level-export-ss-files") - ("impact" . "medium") ("implemented_in" . "") - ("implemented_tool" . "") ("status" . "open") + ("impact" . "medium") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_verify") + ("status" . "implemented") ("tags" "verify" "compile-check" "export" "project-ss" "jerbuild") ("title" @@ -1448,7 +1503,9 @@ . "Validate a single changed Jerboa source file before running a full project build.") ("votes" . 0)) - (("closed_reason" . "") + (("closed_reason" + . + "jerboa_howto now scores exact/tag/title/id matches and tokenized multi-term queries before applying max_results.") ("description" . "jerboa_howto returned no results for queries containing terms that appear in existing recipe titles and tags, while jerboa_howto_get by exact ID retrieved the recipes. Cookbook search should strongly weight exact matches in recipe IDs, titles, and tags, and ideally expose an exact-id lookup fallback in search results.") @@ -1459,8 +1516,9 @@ . "Queries such as \"Ollama OpenAI compatible assistant tool_calls empty content nil\" and \"Ollama tool template malformed XML fallback without tools\" returned no results, even though recipes openai-assistant-tool-call-empty-content and ollama-tool-template-fallback existed with overlapping title/tag terms.") ("id" . "howto-search-exact-tag-title-matches") - ("impact" . "medium") ("implemented_in" . "") - ("implemented_tool" . "") ("status" . "open") + ("impact" . "medium") ("implemented_in" . "mcp/server.ss") + ("implemented_tool" . "jerboa_howto") + ("status" . "implemented") ("tags" "howto" "cookbook" "search" "tags" "ranking") ("title" . @@ -1499,6 +1557,12 @@ "After editing jcode's checkpoints.ss and lsp.ss, compile_check crashed on both with string-ref 8294/11275 == file length. Fell back to make binary (2+ min) to validate.") ("id" . "compile-check-eof-stringref-crash") ("impact" . "high") + ("implemented_in" . "lib/jerboa/translator.sls") + ("implemented_tool" . "jerboa_compile_check") + ("closed_reason" + . + "Fixed the translator scanner EOF condition where escaped string traversal could skip past the requested index and later call string-ref at string length.") + ("status" . "implemented") ("tags" "compile_check" "crash" "eof" "string-ref" "verification") ("title" @@ -1507,4 +1571,107 @@ ("use_case" . "Verifying any .ss edit before building, as mandated by jerboa-* project CLAUDE.md files.") + ("votes" . 0)) + (("description" + . + "The compact Jerboa dispatcher has tools like howto_add, list_features, vote_feature, and suggest_feature, but searching natural phrases such as \"howto add recipe cookbook\" and \"list features suggest feature vote feature\" returned no matches. A catalog search should index operation aliases, CRUD verbs, and related nouns so users can discover the right compact-dispatcher target without falling back to paging the catalog.") + ("estimated_token_reduction" + . + "~300-700 tokens per save-discoveries invocation by avoiding failed searches and extra catalog exploration.") + ("example_scenario" + . + "During save-discoveries, searching for \"howto add recipe cookbook\" returned no tools even though howto_add existed and was visible only after a broader catalog query for \"howto\".") + ("id" . "catalog-search-crud-aliases") ("impact" . "medium") + ("tags" "tool-discovery" "catalog" "search" "howto" + "features") + ("title" + . + "Improve Jerboa tool catalog search for add/list/vote aliases") + ("use_case" + . + "When following a workflow that says to add cookbook recipes or suggest features, the agent needs to discover the exact dispatcher tool names quickly.") + ("votes" . 0)) + (("description" + . + "Debugging jcode hangs required many repeated shell probes: tailing ~/jcode.trace and run logs, grepping for body-len/tokens-in/compaction/no-progress/tool.task, checking port 5555 listeners, tmux sessions, exit files, and process state. A single tool should summarize whether the run is live, hung, exited cleanly, over context budget, repeatedly calling the same tools, or leaking child processes.") + ("estimated_token_reduction" + . + "~2k-6k tokens per live-debug cycle and eliminates 5-8 shell probes per status check.") + ("example_scenario" + . + "During a local MLX run, manual greps showed prompt growth from tokens-in=2509 to 8997 before compaction fixes, then later bounded tokens-in around 2999 with a clean no-progress-break exit. A summary tool could have shown that in one call.") + ("id" . "jcode-trace-health-summary") ("impact" . "high") + ("status" . "open") + ("tags" "jcode" "trace" "hang-debugging" "tokens" + "process-health") + ("title" . "Add a jcode trace health summary tool") + ("use_case" + . + "When a user reports that jcode is stuck on thinking, the TUI is unresponsive, or a monitored local-model run has stopped responding.") + ("votes" . 0)) + (("description" + . + "Long-running debug sessions showed many orphaned jmcp processes left behind after jcode restarts and monitored run exits. jcode should track MCP/LSP child PIDs and terminate/reap them during normal exit, watchdog restarts, and error shutdown paths. The health tooling should also flag stale children whose parent is no longer the active jcode process.") + ("estimated_token_reduction" + . + "~500-1500 tokens per long debug session by reducing process-noise checks; also reduces background resource leaks.") + ("example_scenario" + . + "After multiple monitored runs, ps showed many /Users/user/.local/bin/jmcp processes with PPID 1 plus newer active jmcp/jlsp children. They were not causing the prompt hang directly, but they make live diagnosis noisy and can waste resources.") + ("id" . "jcode-reap-child-processes-on-exit") + ("impact" . "medium") ("status" . "open") + ("tags" "jcode" "process-cleanup" "mcp" "watchdog" + "resource-leak") + ("title" . "Reap jcode child processes on exit and restart") + ("use_case" + . + "Repeatedly restarting jcode under tmux/watchdog while debugging local model hangs or crashes.") + ("votes" . 0)) + (("description" + . + "jerboa_verify and jerboa_compile_check crashed before reporting syntax or compile results on large project files, with errors like `Exception in string-ref: 102556 is not a valid index` for provider.ss and a similar index error for test/run.ss. The tools should either process large files successfully or return a structured diagnostic that identifies the failing phase and file/line context.") + ("estimated_token_reduction" + . + "~300-800 tokens per failed verification by avoiding fallback diagnosis and repeated tool attempts") + ("example_scenario" + . + "After changing src/jcode/provider/provider.ss and test/run.ss, both jerboa_verify and jerboa_compile_check failed internally with invalid string-ref indices, forcing fallback to make build/test for authoritative verification.") + ("id" . "verify-large-file-index-crash") + ("impact" . "medium") + ("implemented_in" . "lib/jerboa/translator.sls") + ("implemented_tool" . "jerboa_verify") + ("note" + . + "The reproduced string-ref index crash is fixed in translator string scanning; broader structured diagnostics remain future hardening.") + ("status" . "partial") + ("tags" + "verify" + "compile_check" + "large-files" + "diagnostics") + ("title" + . + "Make verify and compile_check handle large files without string-ref index crashes") + ("use_case" + . + "Use mandatory Jerboa verification tools on real project modules and large test harness files after edits.") + ("votes" . 0)) + (("description" + . + "When jerboa_verify or jerboa_compile_check is called on a project file, failures can return only \"Unexpected output:\" with an empty payload. jerboa_check_syntax also rejects file_path and requires inline code, which makes fallback validation more awkward than necessary. The tools should report the command run, stderr/stdout snippets, exit status, and accept file_path where practical.") + ("estimated_token_reduction" + . + "~500-1000 tokens per failed verification by avoiding manual fallback commands and repeated diagnosis.") + ("example_scenario" + . + "While editing src/jcode/core/agent.ss and test/run.ss, jerboa_verify and jerboa_compile_check returned empty Unexpected output messages. I had to fall back to check_balance plus make build/test to confirm validity.") + ("id" . "verify-project-file-diagnostics") + ("impact" . "medium") + ("tags" "verify" "compile-check" "diagnostics" "file-path") + ("title" + . + "Improve project-file diagnostics for verify and compile checks") + ("use_case" + . + "Validating changed .ss files in a Jerboa project before running make build.") ("votes" . 0))) --- a/lib/jerboa/prelude.ss +++ b/lib/jerboa/prelude.ss @@ -210,6 +210,15 @@ write-csv write-csv-file rows->csv-string csv->alists alists->csv + ;; ---- std/safe default aliases ---- + sqlite-open sqlite-close sqlite-exec sqlite-execute sqlite-query + sqlite-prepare sqlite-finalize sqlite-step sqlite-bind + tcp-connect tcp-listen tcp-accept tcp-close + tcp-read tcp-write tcp-write-string + open-safe-input-file open-safe-output-file + call-with-safe-input-file call-with-safe-output-file + *safe-mode* + ;; ---- FFI ---- c-lambda define-c-lambda begin-ffi c-declare @@ -303,6 +312,7 @@ (std datetime) (std debug pp) (std csv) + (prefix (std safe) safe:) (std ergo) (std contract condition) (std misc atom) @@ -334,6 +344,33 @@ (def (string-map f s) (list->string (map f (string->list s)))) + ;; ---- Safe defaults ---- + ;; The simple database/network names use contract-checked safe wrappers. + ;; Raw FFI-backed modules remain available through explicit imports. + (def sqlite-open safe:safe-sqlite-open) + (def sqlite-close safe:safe-sqlite-close) + (def sqlite-exec safe:safe-sqlite-exec) + (def sqlite-execute safe:safe-sqlite-execute) + (def sqlite-query safe:safe-sqlite-query) + (def sqlite-prepare safe:safe-sqlite-prepare) + (def sqlite-finalize safe:safe-sqlite-finalize) + (def sqlite-step safe:safe-sqlite-step) + (def sqlite-bind safe:safe-sqlite-bind) + + (def tcp-connect safe:safe-tcp-connect) + (def tcp-listen safe:safe-tcp-listen) + (def tcp-accept safe:safe-tcp-accept) + (def tcp-close safe:safe-tcp-close) + (def tcp-read safe:safe-tcp-read) + (def tcp-write safe:safe-tcp-write) + (def tcp-write-string safe:safe-tcp-write-string) + + (def open-safe-input-file safe:safe-open-input-file) + (def open-safe-output-file safe:safe-open-output-file) + (def call-with-safe-input-file safe:safe-call-with-input-file) + (def call-with-safe-output-file safe:safe-call-with-output-file) + (def *safe-mode* safe:*safe-mode*) + ;; ---- Regex AI compatibility aliases ---- ;; LLMs trained on Python/Ruby/JavaScript commonly use these generic names. (def (regex-match pat str) (re-search pat str)) --- a/lib/jerboa/translator.sls +++ b/lib/jerboa/translator.sls @@ -102,13 +102,38 @@ (char=? ch #\=) (char=? ch #\.) (char=? ch #\@) (char=? ch #\^) (char=? ch #\~) (char=? ch #\%))) + (define (reader-delimiter? ch) + (or (char-whitespace? ch) + (char=? ch #\() + (char=? ch #\)) + (char=? ch #\[) + (char=? ch #\]) + (char=? ch #\{) + (char=? ch #\}) + (char=? ch #\") + (char=? ch #\;) + (char=? ch #\'))) + + (define (char-literal-end str i) + (let* ([len (string-length str)] + [start (+ i 2)]) + (cond + [(>= start len) len] + [(reader-delimiter? (string-ref str start)) (+ start 1)] + [else + (let loop ([j start]) + (if (or (>= j len) (reader-delimiter? (string-ref str j))) + j + (loop (+ j 1))))]))) + ;; Is character at position i inside a string literal? ;; Simple scan from start (does not handle nested/escaped properly for ;; all edge cases, but covers the common case). (define (in-string-at? str i) - (let loop ([j 0] [in-str #f]) + (let ([len (string-length str)]) + (let loop ([j 0] [in-str #f]) (cond - [(= j i) in-str] + [(or (>= j i) (>= j len)) in-str] [(and (not in-str) (char=? (string-ref str j) #\")) (loop (+ j 1) #t)] [(and in-str (char=? (string-ref str j) #\\)) @@ -116,7 +141,7 @@ [(and in-str (char=? (string-ref str j) #\")) (loop (+ j 1) #f)] [else - (loop (+ j 1) in-str)]))) + (loop (+ j 1) in-str)])))) ;; ========== String-level Transformations ========== @@ -232,6 +257,11 @@ (cond [(>= i len) (apply string-append (reverse acc))] + [(and (< (+ i 1) len) + (char=? (string-ref str i) #\#) + (char=? (string-ref str (+ i 1)) #\\)) + (let ([j (char-literal-end str i)]) + (loop j (cons (substring str i j) acc) pctx bstk))] [(in-string-at? str i) (loop (+ i 1) (cons (string (string-ref str i)) acc) pctx bstk)] ;; Open paren: determine context to push @@ -286,6 +316,11 @@ (cond [(>= i len) (apply string-append (reverse acc))] + [(and (< (+ i 1) len) + (char=? (string-ref str i) #\#) + (char=? (string-ref str (+ i 1)) #\\)) + (let ([j (char-literal-end str i)]) + (loop j (cons (substring str i j) acc)))] [(in-string-at? str i) (loop (+ i 1) (cons (string (string-ref str i)) acc))] [(char=? (string-ref str i) #\{) --- a/mcp/server.ss +++ b/mcp/server.ss @@ -81,10 +81,31 @@ override (env "JERBOA_HOME" (path-join (getenv "HOME") "mine" "jerboa")))) +(def (repo-local-scheme-path) + (let ([root (path-join (repo-root) ".chez" "lib")]) + (and (file-directory? root) + (let version-loop ([versions (guard (e [else '()]) (directory-list root))]) + (cond + [(null? versions) #f] + [else + (let ([version-dir (path-join root (car versions))]) + (if (file-directory? version-dir) + (let machine-loop ([machines (guard (e [else '()]) (directory-list version-dir))]) + (cond + [(null? machines) (version-loop (cdr versions))] + [else + (let ([candidate (path-join version-dir (car machines) "scheme")]) + (if (file-exists? candidate) + candidate + (machine-loop (cdr machines))))])) + (version-loop (cdr versions))))]))))) + (def (scheme-path) (def explicit (getenv "JERBOA_MCP_SCHEME_PATH")) + (def repo-scheme (repo-local-scheme-path)) (cond [(and explicit (file-exists? explicit)) explicit] + [repo-scheme repo-scheme] [(file-exists? (path-join (getenv "HOME") "mine" "ChezScheme" "tarm64osx" "bin" "tarm64osx" "scheme")) (path-join (getenv "HOME") "mine" "ChezScheme" "tarm64osx" "bin" "tarm64osx" "scheme")] [(file-exists? (path-join (getenv "HOME") ".local" "bin" "scheme")) @@ -205,6 +226,139 @@ " (loop)))))\n" " (display \"" valid-marker "\\n\"))\n")) +(def (classify-jerbuild-source? file source) + (and file + (string-suffix? ".ss" file) + (not (chez-reader-source? source)) + (not (string-prefix? "(library " (string-trim source))) + (not (null? (hash-ref (parse-source file source) "exports"))))) + +(def (path-remove-source-extension path) + (cond + [(string-suffix? ".ss" path) (substring path 0 (- (string-length path) 3))] + [(string-suffix? ".sls" path) (substring path 0 (- (string-length path) 4))] + [else path])) + +(def (directory-prefix dir) + (let ([trimmed (if dir dir "")]) + (cond + [(= (string-length trimmed) 0) ""] + [(char=? (string-ref trimmed (- (string-length trimmed) 1)) #\/) trimmed] + [else (string-append trimmed "/")]))) + +(def (string-starts-with-chars? s prefix) + (let ([slen (string-length s)] + [plen (string-length prefix)]) + (and (<= plen slen) + (let loop ([i 0]) + (cond + [(= i plen) #t] + [(char=? (string-ref s i) (string-ref prefix i)) (loop (+ i 1))] + [else #f]))))) + +(def (path-relative-under path dir) + (let ([prefix (directory-prefix dir)]) + (and (> (string-length prefix) 0) + (string-starts-with-chars? path prefix) + (substring path (string-length prefix) (string-length path))))) + +(def (jerbuild-library-name-text file project) + (let* ([prefixes (filter (lambda (x) x) + (list (and project (path-join project "src")) + (and project (path-join project "lib")) + project))] + [relative (let loop ([rest prefixes]) + (cond + [(null? rest) (path-strip-directory file)] + [else + (let ([hit (path-relative-under file (car rest))]) + (if hit hit (loop (cdr rest))))]))] + [no-ext (path-remove-source-extension relative)] + [parts (filter (lambda (part) (> (string-length part) 0)) + (string-split no-ext #\/))]) + (string-append "(" (string-join parts " ") ")"))) + +(def (build-jerbuild-module-syntax-script code module-name) + (def escaped (scheme-escape code)) + (string-append + (join-lines (import-lines '())) + "\n(import (jerboa translator))\n\n" + "(define (read-all port)\n" + " (let loop ([out '()])\n" + " (let ([expr (read port)])\n" + " (if (eof-object? expr) (reverse out) (loop (cons expr out))))))\n" + "(define (classify forms)\n" + " (let loop ([forms forms] [exports '()] [imports '()] [body '()])\n" + " (cond\n" + " [(null? forms) (list exports imports (reverse body))]\n" + " [(and (pair? (car forms)) (eq? (caar forms) 'export))\n" + " (loop (cdr forms) (append exports (cdar forms)) imports body)]\n" + " [(and (pair? (car forms)) (eq? (caar forms) 'import))\n" + " (loop (cdr forms) exports (append imports (cdar forms)) body)]\n" + " [(and (pair? (car forms)) (eq? (caar forms) 'declare))\n" + " (loop (cdr forms) exports imports body)]\n" + " [else (loop (cdr forms) exports imports (cons (car forms) body))])))\n\n" + "(guard (e [else\n" + " (display \"" error-marker "\\n\")\n" + " (display-condition e (current-output-port))])\n" + " (let* ([cooked (translate-method-dispatch\n" + " (translate-hash-bang\n" + " (translate-keywords \"" escaped "\")))]\n" + " [xlate (apply make-translator (default-transforms))]\n" + " [translated (map xlate (read-all (open-string-input-port cooked)))]\n" + " [parts (classify translated)]\n" + " [exports (list-ref parts 0)]\n" + " [imports (list-ref parts 1)]\n" + " [body (list-ref parts 2)]\n" + " [library-name '" module-name "]\n" + " [imported-libs (map (lambda (spec)\n" + " (let loop ([x spec])\n" + " (if (and (pair? x)\n" + " (memq (car x) '(only except except-for prefix rename add-prefix drop-prefix))\n" + " (pair? (cdr x)))\n" + " (loop (cadr x))\n" + " x)))\n" + " imports)]\n" + " [has-prelude? (or (member '(jerboa prelude) imported-libs)\n" + " (member '(jerboa prelude clean) imported-libs))]\n" + " [auto-imports (if has-prelude? '() '((jerboa core) (jerboa runtime)))]\n" + " [all-libs (append imported-libs auto-imports)]\n" + " [exclusion-triggers\n" + " '(((jerboa core) . (make-hash-table hash-table? iota 1+ 1- getenv\n" + " path-extension path-absolute?\n" + " thread? make-mutex mutex? mutex-name))\n" + " ((jerboa runtime) . (make-hash-table hash-table? iota 1+ 1-))\n" + " ((std sort) . (sort sort!))\n" + " ((std format) . (printf fprintf))\n" + " ((std os path) . (path-extension path-absolute?))\n" + " ((std misc ports) . (with-input-from-string with-output-to-string))\n" + " ((std datetime) . (make-date make-time))\n" + " ((jerboa prelude) . (make-hash-table hash-table? sort sort! printf fprintf format\n" + " path-extension path-absolute?\n" + " with-input-from-string with-output-to-string\n" + " iota 1+ 1- partition make-date make-time meta atom?))\n" + " ((jerboa prelude clean) . (make-hash-table hash-table? sort sort! printf fprintf format\n" + " path-extension path-absolute?\n" + " with-input-from-string with-output-to-string\n" + " iota 1+ 1- partition make-date make-time meta atom?)))]\n" + " [unique-symbols (lambda (xs)\n" + " (let loop ([rest xs] [seen '()] [out '()])\n" + " (cond [(null? rest) (reverse out)]\n" + " [(memq (car rest) seen) (loop (cdr rest) seen out)]\n" + " [else (loop (cdr rest) (cons (car rest) seen) (cons (car rest) out))])))]\n" + " [candidate-lists (map (lambda (lib)\n" + " (let ([hit (assoc lib exclusion-triggers)])\n" + " (if hit (cdr hit) '())))\n" + " all-libs)]\n" + " [chez-symbols (environment-symbols (environment '(" "chezscheme" ")))]\n" + " [exclusions (filter (lambda (sym) (member sym chez-symbols))\n" + " (unique-symbols (if (null? candidate-lists) '() (apply append candidate-lists))))]\n" + " [chez-import (if (null? exclusions) '(" "chezscheme" ") `(except (" "chezscheme" ") ,@exclusions))]\n" + " [import-form `(import ,chez-import ,@imports ,@auto-imports)]\n" + " [library-form `(library ,library-name (export ,@exports) ,import-form ,@body)])\n" + " (compile library-form))\n" + " (display \"" valid-marker "\\n\"))\n")) + (def (build-library-syntax-script file) (def escaped (scheme-escape file)) (string-append @@ -606,6 +760,19 @@ (def (string-ci-contains? hay needle) (string-contains (string-downcase hay) (string-downcase needle))) +(def (sum-numbers xs) + (if (null? xs) 0 (apply + xs))) + +(def (search-terms query) + (filter (lambda (term) (> (string-length term) 0)) + (string-split (string-trim (string-downcase query)) #\space))) + +(def (score-text text term weight) + (if (string-ci-contains? text term) weight 0)) + +(def (score-tags tags term weight) + (if (any (lambda (tag) (string-ci-contains? tag term)) tags) weight 0)) + (def (alist-record? x) (and (list? x) (not (null? x)) @@ -678,12 +845,44 @@ (if notes (string-append "\n" notes) ""))))) (def (recipe-matches? recipe query) - (def q (string-downcase query)) - (or (string-ci-contains? (recipe-field recipe "id") q) - (string-ci-contains? (recipe-field recipe "title") q) - (string-ci-contains? (recipe-field recipe "code") q) - (any (lambda (tag) (string-ci-contains? tag q)) - (hash-get* recipe "tags" '())))) + (> (recipe-search-score recipe query) 0)) + +(def (recipe-term-score recipe term) + (let ([id (recipe-field recipe "id")] + [title (recipe-field recipe "title")] + [tags (hash-get* recipe "tags" '())] + [notes (hash-get* recipe "notes" "")] + [code (recipe-field recipe "code")]) + (+ (if (string=? (string-downcase id) term) 80 0) + (score-text id term 30) + (score-text title term 25) + (score-tags tags term 20) + (score-text notes term 8) + (score-text code term 2)))) + +(def (recipe-search-score recipe query) + (let* ([literal (string-trim query)] + [terms (search-terms query)] + [literal-score (if (and (> (string-length literal) 0) + (or (string-ci-contains? (recipe-field recipe "id") literal) + (string-ci-contains? (recipe-field recipe "title") literal) + (string-ci-contains? (hash-get* recipe "notes" "") literal) + (any (lambda (tag) (string-ci-contains? tag literal)) + (hash-get* recipe "tags" '())))) + 50 + 0)]) + (+ literal-score + (sum-numbers (map (lambda (term) (recipe-term-score recipe term)) terms))))) + +(def (rank-recipes recipes query) + (sort recipes + (lambda (a b) + (let ([as (recipe-search-score a query)] + [bs (recipe-search-score b query)]) + (cond + [(> as bs) #t] + [(< as bs) #f] + [else (string<? (recipe-field a "id") (recipe-field b "id"))]))))) (def (tool-eval args) (def expression (hash-get* args "expression" #f)) @@ -735,7 +934,15 @@ (let* ([home (hash-get* args "jerboa_home" #f)] [script (if (chez-reader-file? file source) (build-library-syntax-script file) - (build-syntax-script (strip-shebang source) '()))] + (let ([stripped (strip-shebang source)]) + (if (classify-jerbuild-source? file stripped) + (build-jerbuild-module-syntax-script + stripped + (normalize-import + (hash-get* args "module_path" + (jerbuild-library-name-text file + (hash-get* args "project_path" #f))))) + (build-syntax-script stripped '()))))] [out (run-jerboa-script script home (tool-extra-libdirs args file home))] [label (if file file "code")]) (cond @@ -775,7 +982,15 @@ [home (hash-get* args "jerboa_home" #f)] [script (if (chez-reader-file? file source) (build-library-syntax-script file) - (build-syntax-script (strip-shebang source) '()))] + (let ([stripped (strip-shebang source)]) + (if (classify-jerbuild-source? file stripped) + (build-jerbuild-module-syntax-script + stripped + (normalize-import + (hash-get* args "module_path" + (jerbuild-library-name-text file + (hash-get* args "project_path" #f))))) + (build-syntax-script stripped '()))))] [out (run-jerboa-script script home (tool-extra-libdirs args file home))]) (cond [(index-of out valid-marker) @@ -922,7 +1137,8 @@ (if (not query) (text-result "query is required." #t) (let* ([recipes (load-cookbook path)] - [matches (take (filter (lambda (r) (recipe-matches? r query)) recipes) max-results)]) + [matches (take (rank-recipes (filter (lambda (r) (recipe-matches? r query)) recipes) query) + max-results)]) (if (null? matches) (text-result (string-append "No recipes found for \"" query "\".")) (text-result @@ -2394,6 +2610,9 @@ [(member (car rest) seen) (loop (cdr rest) seen out)] [else (loop (cdr rest) (cons (car rest) seen) (cons (car rest) out))]))) +(def (append-all lists) + (if (null? lists) '() (apply append lists))) + (def (set-diff-strings a b) (filter (lambda (x) (not (member x b))) a)) @@ -2457,6 +2676,114 @@ (jhash "ok" #t "module" normalized "entries" (map catalog-line->entry (marker-payloads out catalog-marker)))))) +(def (line-prefix-value lines prefix) + (let ([line (find (lambda (l) (string-prefix? prefix l)) lines)]) + (and line (substring line (string-length prefix) (string-length line))))) + +(def (arity-mask-number arity) + (and (string? arity) + (not (string=? arity "-")) + (string->number arity))) + +(def (arity-mask-first-accepted arity) + (let ([n (arity-mask-number arity)]) + (and n + (let ([m (if (< n 0) (- n) n)]) + (let loop ([i 0] [bit 1]) + (cond + [(> i 8) #f] + [(not (= 0 (bitwise-and m bit))) i] + [else (loop (+ i 1) (ash bit 1))])))))) + +(def (example-args count variadic?) + (let ([args (if count + (map (lambda (i) (string-append "arg" (number->string (+ i 1)))) + (iota count)) + '("arg"))]) + (string-join (if variadic? (append args '("...")) args) " "))) + +(def (arity-call-skeleton symbol kind arity) + (if (or (string=? kind "proc") (string=? kind "procedure")) + (let* ([count (arity-mask-first-accepted arity)] + [n (arity-mask-number arity)] + [variadic? (and n (< n 0))] + [args (example-args count variadic?)]) + (if (> (string-length args) 0) + (string-append "(" symbol " " args ")") + (string-append "(" symbol ")"))) + symbol)) + +(def (curated-usage-example symbol module) + (cond + [(string=? symbol "make-counter") + "(make-counter registry \"requests_total\" \"Total requests\" '(\"method\"))"] + [(string=? symbol "make-gauge") + "(make-gauge registry \"queue_depth\" \"Queued jobs\" '(\"queue\"))"] + [(string=? symbol "make-histogram") + "(make-histogram registry \"request_seconds\" \"Request latency\" '(\"route\") '(0.1 0.5 1.0))"] + [(string=? symbol "hash-ref") + "(hash-ref table key default)"] + [(string=? symbol "hash-put!") + "(hash-put! table key value)"] + [(string=? symbol "string-split") + "(string-split \"a,b,c\" #\\,)"] + [(string=? symbol "sort") + "(sort '(3 1 2) <)"] + [(string=? symbol "in-range") + "(in-range 0 10 2)"] + [(string=? symbol "make-mult") + "(make-mult source-channel 'timeout 25)"] + [(string=? symbol "tap!") + "(tap! mult output-channel)"] + [(string=? symbol "http-fetch") + "(http-fetch \"https://example.com\" method: \"GET\")"] + [(string=? symbol "http-fetch-get") + "(http-fetch-get \"https://example.com\")"] + [(string=? symbol "sqlite-open")