Merge branch 'forge': full forge guardrails port
ober
89d23c66641ed40470bd0c9c02267e20e3d517c3
--- a/README.md +++ b/README.md @@ -1 +1,24 @@ # jerboa-code + +A portable AI coding agent written in Jerboa Scheme (compiled by jerbuild to +Chez). Source lives under `src/jcode/`; entry point is `main.ss`. + +```bash +make build # compile src/ → lib/ +make test # run test/run.ss +make binary # produce the standalone ./jcode +make run # interactive agent REPL +``` + +## Forge guardrails + +jcode embeds a native port of [forge](https://github.com/azambelli/forge) — a +reliability layer that makes small / self-hosted models call tools dependably. +The guardrails (rescue, validation + retry budget, respond-forcing, per-model +sampling, compaction) are **always on for every provider**. The `/forge` +command is the control / status / ablation surface, and `jcode proxy` exposes +the same guardrails over an OpenAI-compatible HTTP endpoint. + +See **[docs/FORGE.md](docs/FORGE.md)** for the command reference, the workflow +engine, the proxy, and the eval/ablation harness. Design notes and forge-source +citations are in [docs/FORGE_PORT_PLAN.md](docs/FORGE_PORT_PLAN.md). --- a/build-binary.ss +++ b/build-binary.ss @@ -110,6 +110,10 @@ '("lib/jcode/core/models" "lib/jcode/core/config" "lib/jcode/core/log" + "lib/jcode/core/errors" + "lib/jcode/core/hardware" + "lib/jcode/core/steps" + "lib/jcode/core/workflow" "lib/jcode/core/session" "lib/jcode/core/message" "lib/jcode/core/secrets" @@ -120,6 +124,7 @@ "lib/jcode/core/agents-md" "lib/jcode/core/hooks" "lib/jcode/core/compaction" + "lib/jcode/core/compaction-strategy" "lib/jcode/core/sandbox" "lib/jcode/core/repomap" "lib/jcode/core/checkpoints" @@ -129,7 +134,24 @@ "lib/jcode/core/debug-repl" "lib/jcode/core/skill" "lib/jcode/core/builtin-skills" + "lib/jcode/guardrails/nudge" + "lib/jcode/guardrails/error-tracker" + "lib/jcode/guardrails/message-type" + "lib/jcode/guardrails/rescue" + "lib/jcode/guardrails/validator" + "lib/jcode/guardrails/respond" + "lib/jcode/guardrails/guardrails" + "lib/jcode/guardrails/step-enforcer" + "lib/jcode/core/workflow-runner" + "lib/jcode/provider/sampling" "lib/jcode/provider/provider" + "lib/jcode/proxy/convert" + "lib/jcode/proxy/handler" + "lib/jcode/core/slot-worker" + "lib/jcode/proxy/server" + "lib/jcode/eval/scenario" + "lib/jcode/eval/ablation" + "lib/jcode/eval/runner" "lib/jcode/tool/registry" "lib/jcode/tool/file" "lib/jcode/tool/apply-patch" @@ -320,6 +342,7 @@ "std/result" "std/datetime" "std/csv" + "std/contract/condition" "std/ergo" "std/misc/string" "std/misc/list" @@ -328,6 +351,7 @@ "std/misc/thread" "std/misc/channel" "std/misc/ports" + "std/misc/process" "std/misc/retry" "std/misc/uuid" "std/misc/atom" new file mode 100644 --- /dev/null +++ b/docs/FORGE.md @@ -0,0 +1,181 @@ +# Forge — reliability guardrails for local LLM tool-calling + +jcode ships a native port of [forge](https://github.com/azambelli/forge), a +reliability layer that makes small / self-hosted models call tools dependably. +It is implemented in 100% Jerboa Scheme (`src/jcode/{guardrails,core,proxy,eval}`) +and is **always on for every provider** — local *and* cloud. The `/forge` +command is therefore a **control / status / ablation** surface, not an on/off +gate, and `jcode proxy` exposes the same guardrails to any OpenAI client. + +For the design rationale and the forge-source citations behind each constant, +see [`FORGE_PORT_PLAN.md`](FORGE_PORT_PLAN.md). + +--- + +## Command reference + +### `/forge` (interactive REPL) + +| Command | Effect | +|---|---| +| `/forge` · `/forge status` | Show guardrail state: rescue, retry budget, respond-forcing, sampling policy, compaction strategy, VRAM tier. | +| `/forge on` · `/forge enforce [on]` | Respond-forcing **ON** — bare text is retried as a tool call (`respond` injected). | +| `/forge off` · `/forge enforce off` | Respond-forcing **OFF** — bare text is a normal final answer. | +| `/forge sampling off` | Apply no per-model sampling params. | +| `/forge sampling on` | Use the model card's sampling profile if known, else backend defaults. | +| `/forge sampling strict` | Use the card profile; error on an unknown model. | +| `/forge workflow` | Describe + self-test the workflow engine with a scripted responder. | +| `/forge proxy` | Describe + self-test the OpenAI-compatible proxy pipeline. | +| `/forge ablation` · `/forge eval` | Describe + self-test the deterministic eval/ablation harness. | + +The `workflow` / `proxy` / `ablation` self-tests run real code against scripted +inputs (no live model), so they double as a smoke test of the whole stack. + +### `jcode proxy` (subcommand) + +```bash +jcode proxy [--port N] [--bind ADDR] # default 127.0.0.1:8080 +``` + +Serves the configured provider behind the guardrails over an OpenAI-compatible +HTTP endpoint. See [The proxy](#the-openai-compatible-proxy) below. + +--- + +## The guardrails + +All of these wrap every provider call automatically. + +- **Rescue** (`guardrails/rescue.ss`) — recover tool calls the model emitted as + prose instead of structured calls: fenced/embedded JSON brace-scan, rehearsal + `name[ARGS]{…}`, Qwen `<function=name>`, Hermes/Qwen3 `<tool_call>`, Mistral + `[TOOL_CALLS]name{…}`, DeepSeek, with `<think>` stripping. +- **Validator** (`guardrails/validator.ss`) — when the model names a tool that + doesn't exist, or returns bare text where a tool was required, inject a nudge + and retry instead of failing. +- **Error budget** (`guardrails/error-tracker.ss`) — after 3 consecutive bad + responses, stop rather than loop forever; tool-execution errors have their own + budget. Nudges are surfaced as **tool results** (`[StepEnforcementError]`, + `[PrerequisiteError]`, `[ToolError]`), because models are pretrained on + "tool failed → try something else". +- **Respond-forcing** (`guardrails/respond.ss`) — optionally inject a synthetic + `respond` tool so a model that wants to answer in prose is kept in + tool-calling mode where the guardrails apply; the reply is unwrapped back to a + normal assistant message. Toggle with `/forge on|off`. +- **Per-model sampling** (`provider/sampling.ss`) — a model card maps each model + to its recommended temperature / top_p / etc. Policy is `off | on | strict`. +- **Compaction** (`core/compaction-strategy.ss`) — pluggable context compaction + (`TieredCompact`, `SlidingWindow`, `NoCompact`, plus jcode's original), always + preserving the system prompt, first user message, and recent steps. + +--- + +## The workflow surface + +`/forge workflow` exercises an optional structured agent loop ported from +forge's `Workflow` + `WorkflowRunner` (`core/workflow.ss`, `core/workflow-runner.ss`). +A workflow constrains the loop with: + +- **required steps** — tools that must run before a terminal tool may fire; +- a **terminal tool** — calling it ends the run and returns its value; +- **prerequisites** — per-tool dependencies (name-only or arg-matched). + +Define one in Jerboa and drive it with a responder (the inference seam): + +```scheme +(def w + (make-workflow + "research" "Search for context, then answer." + (list + (make-tool-def (make-tool-spec "search" "Search." '(("type" . "object"))) + (lambda (args) "results") '()) + (make-tool-def (make-tool-spec "answer" "Final answer." '(("type" . "object"))) + (lambda (args) "done") '("search"))) ; answer requires search + '("search") ; required steps + "answer" ; terminal tool + "You are a research agent.")) + +(run-workflow w "look it up" responder '((max-iterations . 6))) +``` + +The runner enforces premature-terminal nudges (step 3b), prerequisites (3b.2), +batch tool execution (3c–3e), the error budget (3d), and a max-iterations cap +(step 4), raising the matching forge condition when a budget is exhausted. +`core/slot-worker.ss` serializes runs on a single inference slot with +priority + preemption (forge's single-GPU `SlotWorker`). + +--- + +## The OpenAI-compatible proxy + +`jcode proxy` (and the pure, testable `proxy-dispatch` core in +`proxy/server.ss`) speaks the OpenAI chat-completions wire format and runs every +request through the guardrailed handler. + +| Method | Path | Returns | +|---|---|---| +| `GET` | `/health` | `{"status":"ok"}` | +| `GET` | `/v1/models` | OpenAI model list (id `forge`) | +| `POST` | `/v1/chat/completions` | `chat.completion`, or an SSE stream when `stream:true` | + +```bash +jcode proxy --port 8080 & +curl -s localhost:8080/v1/chat/completions \ + -H 'content-type: application/json' \ + -d '{"messages":[{"role":"user","content":"hi"}]}' +``` + +When the client sends `tools`, a `respond` tool is injected so the model stays +in tool-calling mode (where the guardrails apply), then stripped from the reply. +Conversion lives in `proxy/convert.ss`; the guardrailed request handling in +`proxy/handler.ss`. + +--- + +## The eval & ablation harness + +`/forge ablation` runs a **deterministic** harness (`eval/{scenario,ablation,runner}.ss`): +the model is replaced by a scripted responder, so a run depends only on the +script plus the guardrail configuration. This isolates each guardrail's +contribution to outcome quality without a live model. + +- **Scenario** (`eval/scenario.ss`) — a workflow + user message + scripted + responder + a validator; `check-substrings` is forge's case-insensitive, + comma-stripping answer matcher. +- **Ablation presets** (`eval/ablation.ss`): `reforged` (all on), `no_rescue`, + `no_nudge`, `no_steps`, `no_recovery`, `no_compact`, `bare` (all off). +- **Runner** (`eval/runner.ss`) — `run-eval-scenario`, `run-ablation`, + `ablation-pass-rates`, and `analyze-messages` (tool-call / nudge / error / + reasoning tallies). Accuracy = *completeness* ∧ *validate(terminal-args)* ∧ + *validate-state*. + +The built-in demo scripts a model that tries to answer before searching. With +step enforcement **on** it is nudged to search first (accuracy `#t`); with +`no_steps` it answers immediately but ungrounded (accuracy `#f`) — the harness +attributing the quality gain directly to the step-enforcement guardrail: + +``` +reforged complete=#t accuracy=#t iters=3 +no_steps complete=#t accuracy=#f iters=1 +``` + +Only the knobs the runner observes have effect in the deterministic harness +(step-enforcement, max-retries, max-tool-errors); `rescue` / `compaction` live +in the injected responder seam and are recorded for fidelity but do not by +themselves change a scripted run. + +--- + +## Module map + +``` +src/jcode/ + guardrails/ rescue.ss validator.ss error-tracker.ss respond.ss + nudge.ss message-type.ss step-enforcer.ss guardrails.ss + core/ workflow.ss workflow-runner.ss steps.ss slot-worker.ss + compaction-strategy.ss errors.ss (+ message.ss, agent.ss, …) + proxy/ convert.ss handler.ss server.ss + eval/ scenario.ss ablation.ss runner.ss +``` + +All deterministic logic is covered by `test/run.ss` (`make test`). new file mode 100644 --- /dev/null +++ b/docs/FORGE_PORT_PLAN.md @@ -0,0 +1,297 @@ +# Forge → jerboa-code Port Plan + +> **Status: shipped.** All 8 phases are implemented, tested, and merged on the +> `forge` branch. This document is the original design plan, kept for rationale +> and forge-source provenance. For how to *use* the result, see +> [FORGE.md](FORGE.md). + +**Goal.** Reproduce 100% of [forge](../../forge)'s functionality — a reliability layer for self-hosted LLM tool-calling — natively in jerboa-code (Jerboa Scheme), surfaced as a builtin `/forge` command plus always-on guardrail middleware on every provider call. + +**Decisions locked with the user:** +- **Scope: true 100%** — guardrail core + OpenAI-compatible proxy + eval harness. The forge eval *dashboard* is React/Vite/TypeScript and is explicitly **not** ported; we keep the same data (JSONL) and the ASCII/list/markdown report views, which are pure logic. +- **Activation: always-on for all providers** — guardrails wrap every provider (local *and* cloud). The `/forge` command is therefore a **control / status / ablation** surface, not an on/off gate. + +Forge upstream is Python (~6,700 LOC, MIT, Antoine Zambelli). This plan ports behavior, not code; all exact constants/strings below are cited to forge source so the Jerboa implementation can match 1:1. + +--- + +## 1. What jcode already has vs. what forge adds + +This is the load-bearing table. **We extend/unify the left column; we build the right column.** Do not duplicate existing jcode machinery. + +| Forge capability | Already in jcode? | Action | +|---|---|---| +| Rescue-parse tool calls from text | **Partial** — `agent.ss` parses paren-style `tool(k="v")`, Hermes/Qwen3 `<tool_call><function name=…>`, DeepSeek `<｜DSML｜invoke…>`; `provider.ss:641-748` recovers `<tool_call>`/fenced-JSON | **Unify + extend** to forge's full set: fenced/embedded JSON brace-scan, rehearsal `name[ARGS]{…}`, Qwen `<function=name>`, Mistral `[TOOL_CALLS]name{…}`, think-tag stripping | +| Per-model sampling defaults | **Partial** — `apply-mlx-sampling!` hardcodes MLX temp/top_p/rep | **Replace** with full per-model map (3-key identity) + opt-in policy | +| HTTP transient retry (408/429/5xx) | **Yes** — `provider.ss:56-81` `api-call-with-retry` | Keep as-is (orthogonal layer) | +| Context compaction | **Yes but flat** — `compaction.ss`: keep system + last N, stub bulky tool/assistant content over `prune_bytes`, trigger at % of ctx | **Add strategy abstraction** + `TieredCompact` (MessageType-priority 3-phase) + `SlidingWindow` + `NoCompact`; keep current as a 4th strategy | +| Pre/post tool hooks, abort | **Yes** — `hooks.ss`, `registry.ss:99-104` | Reuse as a guardrail hook point | +| Parallel tool exec | **Yes** — `agent.ss execute-tool-calls` (green threads, param re-parameterize) | Reuse | +| Response validation (unknown-tool / bare-text → nudge) | **No** | **Build** (`ResponseValidator`) | +| Retry-with-nudge + error budget | **No** (only HTTP retry) | **Build** (`ErrorTracker`, nudge injection in loop) | +| Synthetic `respond` tool | **No** (jcode treats "no tool calls" as done) | **Build** + resolve coding-loop semantics (§5.A) | +| Step enforcement / `required_steps` / `terminal_tool` / prerequisites | **No** (open-ended loop, no workflow) | **Build** as an *optional* Workflow surface (§5.B) | +| OpenAI-compatible proxy server | **No** (`serve.ss` is JSONL-over-TCP, not HTTP) | **Build** HTTP/1.1 + SSE from scratch on `:std/net/tcp` | +| Eval harness + ablation | **No** | **Build** (scenarios, runner, batch, ablation, metrics, report, significance) | +| Eval React dashboard | n/a | **Out of scope** (keep JSONL + ASCII/markdown) | + +--- + +## 2. Target module layout + +Mirror forge's package structure under `src/jcode/`. New dirs in **bold**. + +``` +src/jcode/ + guardrails/ ← NEW (forge/guardrails + forge/context + forge/tools/respond) + nudge.ss Nudge struct + nudge-text templates (forge/prompts/nudges.py) + rescue.ss unified rescue parser (forge/prompts/templates.py + jcode's existing parsers) + validator.ss ResponseValidator (forge/guardrails/response_validator.py) + step-enforcer.ss StepEnforcer + StepTracker + prerequisites (forge/guardrails/step_enforcer.py, core/steps.py) + error-tracker.ss ErrorTracker (forge/guardrails/error_tracker.py) + guardrails.ss Guardrails facade: check()/record() (forge/guardrails/guardrails.py) + respond.ss synthetic respond tool (forge/tools/respond.py) + compaction-strategy.ss CompactStrategy / NoCompact / SlidingWindow / TieredCompact (forge/context/strategies.py) + hardware.ss detect-hardware + budget tiers (forge/context/hardware.py + server.py VRAM tiers) + provider/ + provider.ss EXTEND: wire validator+rescue+error-budget+respond into chat path + sampling.ss ← NEW MODEL_SAMPLING_DEFAULTS map + get/apply policy (forge/clients/sampling_defaults.py) + core/ + agent.ss EXTEND: guardrail middleware in agent-loop / agent-loop-stream + workflow.ss ← NEW Workflow/ToolSpec/ToolDef/ToolCall structs + validation (forge/core/workflow.py) + workflow-runner.ss ← NEW WorkflowRunner loop (forge/core/runner.py) + SlotWorker (forge/core/slot_worker.py) + guardrails-config.ss ← NEW /forge toggles, ablation state, persisted in jcode.json + proxy/ ← NEW (forge/proxy/* + forge/server.py) + http.ss HTTP/1.1 parse + SSE chunked framing on :std/net/tcp + convert.ss OpenAI ↔ jcode message conversion (forge/proxy/convert.py) + handler.ss per-request guardrail bridge (forge/proxy/handler.py) + server.ss routes, serialize worker, disconnect-cancel (forge/proxy/server.py) + backend.ss ServerManager + BudgetMode (forge/server.py) + eval/ ← NEW (forge/tests/eval/*) — in src so /forge eval compiles into the binary + scenario.ss EvalScenario contract + _check/_validate helpers (_base.py) + scenarios/*.ss the 30 scenarios + runner.ss batch.ss ablation.ss metrics.ss report.ss significance.ss + ui/cli.ss EXTEND: /forge command dispatch in handle-command +``` + +**Layout decisions to confirm during impl:** (a) eval under `src/` (compiled, invokable via `/forge eval`) vs `test/` (forge's choice) — recommend `src/jcode/eval/`; (b) `respond` registered through existing `register-internal-tool!` so it never leaks into normal tool listings. + +--- + +## 3. Faithful behavior spec (the 1:1 contract) + +Everything an implementer needs to match forge exactly. **Cite = forge path:line.** + +### 3.1 Magic constants (quote-exact) + +| Constant | Value | Cite | +|---|---|---| +| WorkflowRunner.max_iterations | **10** (eval scenarios override to 15) | runner.py:36 / eval `_base.py` | +| max_retries_per_step | **3** (eval: 5) | runner.py:37 | +| max_tool_errors | **2** | runner.py:38 / error_tracker.py:19 | +| max_premature_attempts | **3** | step_enforcer.py:49 | +| max_prereq_violations | **2** | step_enforcer.py:49 | +| step nudge tier | `min(attempts, 3)` (1=polite,2=direct,3=aggressive) | step_enforcer.py:77 | +| TieredCompact TRUNCATE_CHARS | **200** | strategies.py:108 | +| TieredCompact keep_recent | **2** | strategies.py:110 | +| compact_threshold | **0.75** of budget | strategies.py:113 | +| token estimate | `chars // 4` | strategies.py:11 | +| context warning tiers | ≥0.80 / ≥0.65 / else | manager.py:33,40 | +| attempt_limit | `max_retries + 1`, clamped to `max_iterations − iteration` | inference.py:163 | +| call_id format | `call_{counter:09d}` (runner) / `call_<8hex>` (proxy) | inference.py:101 / convert.py | +| VRAM budget tiers | ≥48GB→**262144**, ≥24GB→**32768**, else→**4096** | server.py:383 | +| no-hardware fallback | **4096** tokens | hardware.py:80 | +| proxy default host/port | **127.0.0.1 / 8081**; backend **8080** | __main__.py | +| proxy max body | **16 MiB** → 413 | server.py:23 | + +**Exhaustion is strict `>`** everywhere: `max_retries=3` permits 3 failures, trips on the **4th**. Same for premature/prereq/tool-errors. (error_tracker.py:56) + +### 3.2 Nudge text (verbatim — nudges.py) + +- **retry** (bare text): `"Your previous response was not a valid tool call. You must respond with a tool call, not free text. Please try again with a valid tool call."` +- **unknown_tool**: `"Tool '{name}' does not exist. Available tools: {list}. Call one of them."` +- **step tier 1**: `"You cannot call {terminal} yet. You must first complete these required steps: {steps}. Call one of them now."` +- **step tier 2**: `"You must call one of these tools now: {steps}. Pick one."` +- **step tier 3**: `"STOP. You MUST call one of: {steps}. Do NOT call {terminal}. Your next response MUST be a tool call to one of: {steps}."` +- **prerequisite**: `"You cannot call {tool} yet. You must first call: {prereqs}. Call the prerequisite tool now."` + +Nudge struct: `role` (`"user"`|`"tool"`), `content`, `kind` (`retry`/`unknown_tool`/`step`/`prerequisite`), `tier` (0 default). Wire-shape rule: **retry/unknown nudges go on the `user` channel** (model needs a positive instruction); **step/prereq nudges go on the `tool`-error channel** with `[StepEnforcementError]`/`[PrereqError]`/`[UnknownTool]` prefixes (OpenAI-tool-trained models pattern-match "your call failed, retry" better there). (ADR rationale in ARCHITECTURE.md "Guardrails" table; inference.py:245-292) + +### 3.3 Rescue parsing (templates.py) — try strategies **in order**, stop at first hit + +0. **Strip think tags** first: `[THINK]…[/THINK]` and `<think>…</think>` (DOTALL). If empty after, return none. +1. **Embedded/fenced JSON**: strip ```` ```json ```` fences, brace-scan for balanced `{…}`, parse each. Accept forge-style `{"tool","args"}` **or** OpenAI-style `{"name","arguments"}`; tool name must be in the known set. (templates.py:54-114) +2. **Rehearsal**: regex `(\w+)\[ARGS\](\{.*\})` DOTALL, args = JSON. Only if (1) found nothing. (templates.py:120,266) +3. **Qwen Coder XML**: `<function=NAME>` … `<parameter=KEY>value</parameter>` … `</function>`; strip one leading + one trailing newline per value; values stay strings (coerced later). (templates.py:137-187) +4. **Mistral bracket-tag**: `[TOOL_CALLS]NAME` then brace-balanced JSON (string-aware: respects `\` escapes and `"`). (templates.py:152-240) + +jcode already has parsers for #1(partial)/#3 plus paren-style and DeepSeek-invoke that forge lacks — **keep those, add #2 and #4, align #1's dual-key acceptance.** Consolidate all into `guardrails/rescue.ss` so `agent.ss` and the proxy share one parser (forge principle #4: rescue lives at the client/abstraction boundary, not in the loop). + +### 3.4 Synthetic `respond` tool (respond.py) + +- name `"respond"`; single param `message: string` ("The message to send to the user."). +- description (verbatim): `"Respond to the user with a message. Use this when the user is chatting, asking a question, when you need to ask a clarifying question before proceeding, or when no other tool action is needed. Also use this after completing the user's request to report the result."` +- Behavior: injected when tools are present and no `respond` already exists; a pure-`respond` response is **unwrapped to a normal assistant text message** before returning; mixed batches emit only the non-respond calls. (handler.py:112,165) + +### 3.5 ErrorTracker / StepEnforcer state machine (runner.py loop order) + +Per iteration (`while iteration < max_iterations`): +1. **Cancellation** check (once, before inference) → raise `WorkflowCancelledError`. +2. **run_inference** (compact → threshold-check → fold/serialize → inject transient `user` context-warning → send → sync token count → validate; retry-with-nudge internally up to `attempt_limit`). `iteration += attempts` (**retries consume the iteration budget**). Returns `None` when exhausted → `break` → `MaxIterationsError`. +3. **TextResponse** → emit assistant text, `continue`. +4. **Premature-terminal**: if a terminal tool is in the batch and required steps unsatisfied → escalating step nudge on tool channel; raise `StepEnforcementError` past limit. +5. **Prerequisites**: whole-batch blocking — any one violating call blocks the batch; raise `PrerequisiteError` past limit. +6. **Execute batch** — 3 outcomes per call: (a) `ToolResolutionError` is **privileged** — feeds back as `[ToolResolutionError]`, does *not* count against error budget, does *not* record the step; (b) other exception → `[ToolError]`, counts; (c) success → `StepEnforcer.record(tool,args)`, emit result. +7. **Post-batch**: clean batch → reset error/premature/prereq counters; else `record_result(false)`, raise `ToolExecutionError` past limit. +8. **Terminal return**: if a terminal tool succeeded (non-exception result) → return it. + +Control-flow state (`StepTracker.completed_steps`/`executed_tools`, error counters, premature/prereq counters, `iteration`, `tool_call_counter`) lives **on the runner, never in message history** (forge principle #3) so compaction can't corrupt it. + +### 3.6 Compaction strategies (strategies.py) + +`CompactStrategy.compact(messages, budget, step_hint) -> (messages, phase)`; phase 0 = untouched (return the *same* list object so callers' identity check short-circuits). Invariant: never cut `messages[0]` (system) or `messages[1]` (first user). Eligible window = indices `[2, eligible_end)`; `eligible_end` found by counting **distinct consecutive `step_index` values** and keeping the last `keep_recent` iterations (handles variable-size parallel batches). + +**TieredCompact 3-phase priority (exact):** + +| Phase | Drop entirely (in window) | Special | +|---|---|---| +| 1 | step/prereq/retry nudges | tool_result >200 chars → truncate to `[:200]` + `"\n[Truncated — {n} chars removed]"` | +| 2 | + tool_result (whole) | reasoning & text_response **preserved** | +| 3 | + reasoning + text_response | only tool_call skeleton remains | + +Each phase rebuilds from the *original* list (not cumulative) and stops as soon as the estimate drops below the next phase's trigger. Reasoning surviving through phase 2 is the key design choice (the model's interpretation of results matters more than raw results). jcode's current `compaction.ss` ≈ a SlidingWindow+truncate hybrid → keep it as `JcodeLegacyCompact`; default switches to `TieredCompact` once `MessageType` tags exist (§6 phase 1 adds them). + +### 3.7 Sampling defaults (sampling_defaults.py) + +Port `MODEL_SAMPLING_DEFAULTS` **verbatim** (every row, all 3 identity keys per model, inline HF-card URL comments preserved). Two functions: +- `get-sampling-defaults(model)` → fresh copy or `{}` (pure lookup, no logging/raising). +- `apply-sampling-defaults(model, strict)` → 4-quadrant: strict+known→dict; strict+unknown→**raise UnsupportedModelError**; non-strict+known→one-shot INFO log, `{}`; non-strict+unknown→`{}`. + +**Always-on interaction:** because guardrails wrap *all* providers, default to **non-strict** (apply if known, else fall through to backend defaults — never break an unknown model). `/forge sampling strict` opts into strict per-session. This supersedes `apply-mlx-sampling!` (MLX values become rows in the map, or remain a provider default if no card exists). + +### 3.8 Errors (errors.py) + +Port the hierarchy: base `ForgeError` → `UnsupportedModelError`, `ToolCallError(raw_response)`, `ToolExecutionError`, `WorkflowCancelledError`, `MaxIterationsError`, `StepEnforcementError`, `PrerequisiteError`, `ContextBudgetExceeded`, `HardwareDetectionError`, `ContextDiscoveryError`, `BudgetResolutionError`, `BackendError`→`ThinkingNotSupportedError`, `StreamError`. **`ToolResolutionError` is deliberately NOT a ForgeError** (plain exception; the privileged "retry with different args" feedback). Map to Jerboa conditions; watch the `else`-in-`guard` gotcha (see §7). + +--- + +## 4. The `/forge` command (control surface) + +Because guardrails are always-on, `/forge` is stateful control — implement as a real case in `cli.ss handle-command` (lines 283-388), **not** a builtin-skill prompt string (those just feed text to `agent-run`). State persists under a `"forge"` block in `jcode.json` via `guardrails-config.ss`. + +| Subcommand | Action | +|---|---| +| `/forge` or `/forge status` | Show active guardrails, current ablation, retry/error budgets, compaction strategy, detected hardware + budget, proxy state | +| `/forge ablation <preset>` | Apply an ablation preset (reforged/no_rescue/no_nudge/no_steps/no_recovery/no_compact/bare) to the live session — directly reuses eval ablation wiring | +| `/forge rescue on\|off`, `/forge respond on\|off`, `/forge compact <tiered\|sliding\|legacy\|none>`, `/forge retries N`, `/forge budget <mode\|N>` | Per-guardrail toggles/tunes | +| `/forge sampling <off\|on\|strict>` | Sampling-map policy for this session | +| `/forge proxy start [--port …] [--backend …]` / `stop` | Start/stop the OpenAI proxy (§5.C) | +| `/forge eval [--scenario … \|--tags …] [--ablation …] [--runs N]` | Run the eval harness (§5.D), print report | +| `/forge workflow <file.ss>` | Load + run a Workflow definition through WorkflowRunner (§5.B) | + +--- + +## 5. Surfaces (and the workflow-vs-coding-loop resolution) + +### 5.A In-loop guardrail middleware (always-on) + +Wrap `agent.ss` `agent-loop` / `agent-loop-stream`. After the provider returns and tool calls are parsed (the existing 4-format parse stays, now delegating to `rescue.ss`): + +1. `validator.validate(response)` → if bare text and rescue finds a call, use it; if unknown tool or unrescuable text → nudge + `error-tracker.record-retry`; re-infer until success or `retries_exhausted` → raise `ToolCallError`. +2. Inject the nudge on the correct channel (user vs tool) and loop **without** counting an HTTP round as "done". +3. On clean batch, `error-tracker.reset`. + +**Resolving "no terminal tool" in an open coding loop:** jcode's loop ends when the model emits no tool calls (bare text = the answer). Forge forbids bare text via `respond`. Reconcile: +- With guardrails on, **register `respond` as an internal tool** and treat it as the terminal for the *chat* turn. A local model emitting bare text gets **one** rescue/nudge toward `respond`; a `respond` call is unwrapped to the normal assistant message the TUI already renders. +- `required_steps`/`prerequisites` are **empty** in the default coding loop (no fixed workflow), so `StepEnforcer` is a no-op there — exactly forge's "guardrails apply with zero required steps too." Premature-terminal logic only bites when a Workflow is loaded (§5.B). +- Cloud models (Anthropic/OpenAI) reliably choose text vs tool; per the always-on decision they still pass through, but `respond` is effectively never needed — keep it injected for uniformity, cheap. + +### 5.B WorkflowRunner surface (forge's structured workflows) + +For the forge features that have no home in an open loop, port the Workflow concept as an **opt-in** runner (`/forge workflow file.ss`): +- `workflow.ss`: `ToolSpec`/`ToolDef`(+`prerequisites`)/`ToolCall`/`Workflow` structs; `Workflow` validation (every tool key == spec name; required_steps ⊂ tools; terminal ⊂ tools; **terminal ∉ required_steps**; prereqs resolve). Tool params are **JSON-schema dicts** (Jerboa has no Pydantic) — port `from_json_schema`/`get_json_schema`. +- `workflow-runner.ss`: the full loop from §3.5 with step/prereq enforcement + `SlotWorker` (priority queue, lower int = higher priority, default 0 = FIFO, auto-preempt lower-priority running task by setting its cancel event). Streaming + `on_message` map to jcode's existing stream/tool callbacks (`make-stream-cb`/`make-tool-cb` in `serve.ss`). + +This keeps the open coding agent and the structured runner as two front-ends over one shared guardrail core (`guardrails.ss`) — mirroring forge's "middleware is the foundation; proxy and runner compose it." + +### 5.C OpenAI-compatible proxy (forge/proxy + server.py) + +`serve.ss`/`relay.ss` are JSONL-over-raw-TCP with a bespoke schema — **no HTTP, no SSE, no OpenAI shape reusable.** Build on `:std/net/tcp` from scratch, reusing only serve.ss's accept-loop / auth / arg-parse / cancellation *patterns*. + +- **Routes:** `GET /health` → `{"status":"ok"}`; `GET /v1/models` → `{"object":"list","data":[{"id":"forge","object":"model"}]}`; `POST /v1/chat/completions` (JSON or SSE); `OPTIONS *` → 204 + CORS; else 404. +- **Per-request order** (handler.py): convert OpenAI→messages + extract `ToolSpec`s → **inject `respond`** (if tools present, none already) → no-tools fast path forwards untouched (no guardrails) → run the validate/rescue/retry loop (`attempt_limit = max_retries+1`) → **strip `respond`** (pure-respond → text+`finish_reason:"stop"`; mixed → only real calls) → `ToolCallError` caught returns last raw text, not a 5xx. +- **Response shapes** (convert.py): `chatcmpl-<12hex>` ids, `call_<8hex>` tool-call ids, `arguments` **JSON-stringified**, usage zeros. Tool-call finish_reason `"tool_calls"`, text `"stop"`. +- **SSE** (server.py:310-340): send headers immediately (liveness), then HTTP chunked frames `data: <json>\n\n`, terminate `data: [DONE]` + `0\r\n\r\n`. Note forge materializes the whole response then chunks it (no true token streaming) — match unless we wire jcode's real streaming. +- **CLI flags** (`/forge proxy …` and a `jcode proxy` subcommand): `--backend-url` (external) XOR `--backend {llamaserver,llamafile,ollama}` (managed); `--host`/`--port`(8081)/`--backend-port`(8080); `--max-retries`(3); `--no-rescue`; `--budget-mode {backend,manual,forge-full,forge-fast}`; `--budget-tokens`; `--serialize`/`--no-serialize` (auto: managed→on, external→off); `--model`/`--gguf`. +- **BudgetMode** (server.py:26): backend (trust `/props`), manual (`-c N`), forge-full (VRAM tier or `/props` max), forge-fast (half of full). VRAM tiers per §3.1. +- **ServerManager** (backend.ss): managed-mode boot of `llama-server -m GGUF -ngl 999 --port P [--jinja] [-c N] …`; health via polling `/props` for `default_generation_settings` (NOT `/health`), 180s/2s; stop terminates then **sleeps 3s** for VRAM release; ollama uses `ollama stop <model>`. +- **Concurrency:** `--serialize` runs a single inference worker (single-GPU); disconnect-cancel polls writer-closing every 1s. Use `with-mutex` around the shared transcoded socket port (see §7 concurrency gotcha). + +### 5.D Eval harness (forge/tests/eval) + +Port as pure logic; reconcile counts (30 registered = 26 documented [OG-18 + 8 advanced_reasoning] + 4 `compaction_chain_*`). +- **`scenario.ss`** — `EvalScenario`: `name, description, workflow, user_message, budget_tokens=8192, max_iterations=15, max_retries_per_step=5, max_tool_errors=2, validate(args)->bool|#f, validate_state()->bool|#f, build_workflow()->（workflow . validate_state), tags=[], ideal_iterations`. Grading = `validate(captured terminal args) AND validate_state()`; terminal tool wrapped in a capturing closure. `_check(text, substrings)` = lowercase + strip commas + AND-contains. Stateful scenarios use a mutable record (Jerboa `defstruct` + setters) where forge uses a closure-captured DB. +- **Scoring:** `completeness` = runner returned without raising; `accuracy` = grader true (or `#f`/none); `iterations_used` = provider `send` count (wrap provider in a counting shim); `StreamError` → up to 2 fresh re-runs. +- **`ablation.ss`** — 7 presets, exact disables: `reforged`(none), `no_rescue`(rescue off), `no_nudge`(rescue off + retries 0), `no_steps`(required_steps→[]), `no_recovery`(max_tool_errors 0), `no_compact`(force NoCompact, skip compaction scenarios), `bare`(all off). These directly back `/forge ablation`. +- **`metrics.ss`/`report.ss`** — per-scenario completion/correctness/iterations/elapsed/nudges/tool-errors/compaction-phases/wasted-calls; ConfigMetrics score/accuracy/completeness/efficiency/speed; **ASCII table + list + progress + `--markdown` (7 views)** — all portable. **`--html` dashboard NOT ported.** +- **`significance.ss`** — pooled McNemar (exact binomial when discordant ≤25, else continuity-corrected χ²) + Wilson 95% CI, paired on `(scenario,run)` vs reforged. Pure `math` (needs `erfc`, `log1p`) — verify Jerboa stdlib has these or port them. +- **`runner.ss`/`batch.ss`** — single + batch (JSONL append, resume by counting existing `model|backend|mode|ablation|tool_choice|scenario` rows). JSONL row shape per the inventory (model…cost_usd). + +--- + +## 6. Phased roadmap + +Each phase is independently buildable + testable (`make build` → `make binary` → `./jcode …`). Ship guardrails before workflow/proxy/eval. + +**Phase 0 — Scaffolding.** Create `guardrails/` dir + `nudge.ss`, `error-tracker.ss`, errors. Port nudge text + ErrorTracker (pure, trivial unit tests). No loop changes yet. + +**Phase 1 — Message typing + rescue unification.** Add a `MessageType`-equivalent tag to jcode messages (needed by TieredCompact + nudge classification). Consolidate jcode's existing parsers + forge strategies #2/#4 into `guardrails/rescue.ss`; point `agent.ss` and (later) the proxy at it. Unit-test each rescue format against fixtures. + +**Phase 2 — Validator + in-loop retry/respond (the core lift).** `validator.ss`, `respond.ss`, `guardrails.ss`. Wire into `agent.ss agent-loop`/`agent-loop-stream`: validate → nudge → re-infer → error budget; register `respond` internal tool; unwrap respond. This is where always-on guardrails become real for the coding loop. Validate on a local model (ollama/mlx) end-to-end. + +**Phase 3 — Sampling map.** `provider/sampling.ss` with the full verbatim map + policy; replace `apply-mlx-sampling!`; non-strict default. Unit-test lookup + 4-quadrant policy. + +**Phase 4 — Compaction strategies.** `compaction-strategy.ss` (NoCompact/Sliding/Tiered + JcodeLegacy); `hardware.ss` (NVIDIA `nvidia-smi`, AMD sysfs, tiers). Default to Tiered. Test phase transitions against synthetic histories. + +**Phase 5 — Workflow surface.** `workflow.ss` + `workflow-runner.ss` (+SlotWorker). `/forge workflow`. Port forge's step/prereq/runner unit tests. + +**Phase 6 — Proxy.** `proxy/http.ss` (HTTP/1.1 + SSE) → `convert.ss` → `handler.ss` → `server.ss` → `backend.ss`. `/forge proxy` + `jcode proxy`. Test with `curl` (JSON + SSE) and by pointing an external OpenAI client at it. + +**Phase 7 — Eval harness.** `eval/*` + scenarios. `/forge eval`. Reproduce a forge eval row on one model+backend and compare numbers. `significance.ss` last (needs `erfc`). + +**`/forge` command** lands incrementally: status in Phase 2, ablation in Phase 7 (or stub earlier), proxy/eval subcommands as those phases complete. + +--- + +## 7. Jerboa implementation notes (from project memory — non-negotiable) + +- **Write `.ss`, never `.sls`.** Use `(jerboa prelude)`, `def`, `defstruct`, `[...]`/`{...}`, `keyword:` args, `(std …)` modules. Edit `.ss` only via `jerboa_balanced_replace` (dry-run → apply → `jerboa_check_balance`), never raw `Edit`. Verify with `jerboa_compile_check` before declaring done. +- **Build gotchas:** forward references fail in generated `.sls` — define before use. The `else` keyword is captured by `guard` — don't use `else` as a guard clause (bites the errors port directly). Always `(import (std misc string))` for `string-prefix?`/`string-join`/`string-split`. +- **Plugin/binary boot deps:** every transitive `(jerboa prelude)` dependency of new modules must be added to `build-binary.ss` `external-libs` — WPO inlining is not enough. New dirs (`guardrails/`, `proxy/`, `eval/`) need their modules registered there. +- **Concurrency:** a shared transcoded port across green threads corrupts framing — wrap each send+read pair (proxy sockets, any shared stream) in `with-mutex` (`chez-make-mutex`). Chez parameters **don't inherit on spawn** — every spawned thread that logs or serves must re-`parameterize` `current-error-port`/`current-log-level` (the proxy worker + parallel tool exec; `agent.ss execute-tool-calls` already shows the pattern). Blocking FFI/socket calls must declare `__collect_safe` or they pin the thread mutex and starve other green threads. +- **Sleep:** use `thread-sleep!`, not `(sleep (make-time …))` (the prelude shadows `make-time`; it raises silently). Relevant to ServerManager's 3s VRAM wait and health polling. +- **max_tokens:** local provider bodies must set `max_tokens` (jcode already sets 32768 OpenAI / 8192 Anthropic) — keep it; truncated streams break the parser (prior MLX/qwen3 bug). + +--- + +## 8. Testing & acceptance + +- **Unit tests** (`test/run.ss`): forge ships ~865 deterministic tests with no backend. Port the guardrail/validator/step/rescue/compaction/significance suites — they're pure logic and are the fidelity contract. Target: every constant/nudge/phase in §3 has a test. +- **Integration:** `make binary` + `./jcode --help`, then a real local-model session (ollama/mlx) exercising rescue + respond + retry; `curl` against the proxy (JSON + SSE); `/forge eval --scenario basic_2step --runs 5`. +- **Acceptance (the 100% bar):** reproduce a forge eval config (one model + backend + ablation grid) and compare score/completeness/efficiency to forge's JSONL within noise; McNemar significance vs `bare` should reproduce forge's per-guardrail contribution direction. +- **Never commit `jcode.json`** (live keys); scan `git diff --cached` for `sk-`/`AIza`/`Bearer ` before any commit; test the binary before committing (project rule, stated 3+ times). + +--- + +## 9. Open questions / risks + +1. **respond in a coding agent (§5.A).** Forcing `respond` is forge's biggest small-model win, but jcode's TUI/markdown render assumes bare-text finals. Plan: unwrap respond → normal assistant message. Risk: double-handling streaming text. Validate early in Phase 2. +2. **Always-on for cloud models.** Per the decision, frontier models also get the stack. `respond` injection + validate is cheap and harmless, but the retry/nudge loop must not fight a model that legitimately wants to answer in text — the rescue-first-then-respond path covers this. Watch for added latency; `/forge ablation no_nudge` is the escape hatch. +3. **Workflow features vs open loop.** required_steps/prereqs/terminal only meaningful under `/forge workflow`; in the default loop they're inert. Confirm that's the intended "100%" reading (functionality present and reachable, not forced onto every coding turn). +4. **Proxy from scratch.** No HTTP/SSE in jcode today — most net-new code; budget Phase 6 accordingly. Decide whether to expose true token streaming (jcode has it) or match forge's materialize-then-chunk. +5. **`erfc`/`log1p` availability** for significance — confirm in Jerboa stdlib or port the numerics. +6. **Eval location** (`src/` vs `test/`) and **scenario count** doc drift (forge's `report.py` references dead legacy scenario names) — adopt the live 30 and document the 26+4 split. + +--- + +*Provenance: forge `src/forge/{guardrails,core,clients,context,prompts,tools,proxy}/*.py`, `server.py`, `errors.py`, `tests/eval/*`, `docs/ARCHITECTURE.md`, ADRs 005/006/011/012/013/014. jcode integration points: `cli.ss:283-388`, `agent.ss` (agent-loop ~1066, execute-tool-calls ~1167, system-prompt ~37), `provider.ss` (openai-body ~772, rescue ~641-748, sampling ~571, retry ~56-81), `registry.ss:61-104`, `compaction.ss`, `hooks.ss`, `config.ss`, `models.ss`, `serve.ss`.* --- a/src/jcode/core/agent.ss +++ b/src/jcode/core/agent.ss @@ -9,6 +9,7 @@ current-provider-override current-model-override get-current-provider + forge-respond-enforced? try-parse-text-tool-calls try-parse-xml-tool-calls) @@ -24,8 +25,13 @@ ./mentions ./agents-md ./compaction + ./compaction-strategy + ./models :jcode/provider/provider :jcode/tool/registry + :jcode/guardrails/guardrails + :jcode/guardrails/nudge + :jcode/guardrails/respond :jerboa/core :jerboa/runtime) @@ -34,6 +40,14 @@ (def current-provider-override (make-parameter #f)) (def current-model-override (make-parameter #f)) +;; Forge guardrail policy. The guardrail layer (unknown-tool nudge + retry +;; budget + respond unwrap) is always-on for every provider. This parameter +;; gates only the one behavior that would change UX for well-behaved cloud +;; models: treating a clean bare-text response as a failed turn that must be +;; retried as a tool call. Off by default (bare text is a normal final); +;; turn on for small local models that can't be trusted to pick tool-vs-text. +(def forge-respond-enforced? (make-parameter #f)) + (def (system-prompt) (format "You are an expert AI coding assistant. You help users with software development tasks. Working directory: ~a @@ -118,7 +132,10 @@ Be concise. Prefer edit over write for modifying existing files. (else (cons fresh messages)))) (mdl (or (current-model-override) (config-ref "model") ""))) (cond - ((should-compact? rebuilt mdl) (compact-messages rebuilt)) + ((should-compact? rebuilt mdl) + ;; Dispatch to the configured strategy (default Tiered). Budget is the + ;; model's context window; should-compact? already guaranteed it is set. + (run-configured-compaction rebuilt (model-context-window mdl))) (else rebuilt)))) (def (truncated-dir) @@ -1055,15 +1072,21 @@ Be concise. Prefer edit over write for modifying existing files. ;; message without matching tool result messages, which the ;; OpenAI-style API rejects with a 400. (try (session-repair-orphan-tool-calls! session-id) (catch (_) (void))) + ;; In respond-forcing mode, expose the synthetic respond tool so the model + ;; has a structured way to answer the user while staying in tool-calling mode. + (when (forge-respond-enforced?) (register-respond-tool!)) (let ((existing (session-get-messages session-id))) (when (null? existing) (session-add-message session-id (make-system-message (system-prompt))))) (session-add-message session-id (make-user-message (expand-mentions user-input))) - (if (current-stream-cb) - (agent-loop-stream session-id (session-get-messages session-id) 0) - (agent-loop session-id (session-get-messages session-id) 0))) + ;; One guardrails instance per user turn — its retry/error budget persists + ;; across the tool-call rounds of this turn, then resets for the next. + (let ((gr (make-guardrails (list-tools)))) + (if (current-stream-cb) + (agent-loop-stream session-id (session-get-messages session-id) 0 gr) + (agent-loop session-id (session-get-messages session-id) 0 gr)))) -(def (agent-loop session-id messages round) +(def (agent-loop session-id messages round gr) (let* ((provider (get-current-provider)) (tools (get-tool-schemas)) (msgs (refresh-system-prompt messages)) @@ -1090,25 +1113,87 @@ Be concise. Prefer edit over write for modifying existing files. hermes-tcs))) (else response))) (tcs (or (message-tool-calls effective) '()))) - (session-add-message session-id effective) (cond - ((null? tcs) effective) - ((>= round *max-tool-rounds*) - (log-warn logger "max-rounds" `((round . ,round))) - (let ((results (execute-tool-calls tcs))) - (for-each (lambda (r) (session-add-message session-id r)) results) - (let* ((final-msgs (refresh-system-prompt (session-get-messages session-id))) - (final (chat-with-expert provider final-msgs '()))) - (session-add-message session-id final) - final))) + ;; Bare text with respond-forcing off: a normal terminal answer. + ((and (null? tcs) (not (forge-respond-enforced?))) + (session-add-message session-id effective) + effective) (else - (let ((results (execute-tool-calls tcs))) - (for-each - (lambda (result) (session-add-message session-id result)) - results) - (agent-loop session-id (session-get-messages session-id) (+ round 1)))))))) + (agent-guardrails-step + session-id provider effective content tcs round gr)))))) + +;; Apply the guardrail verdict to a (possibly tool-calling) response, then +;; act: stop on fatal, inject a corrective signal on retry, unwrap respond() +;; into a terminal answer, or execute the validated calls and recurse. Each +;; branch persists exactly one assistant message so tool_calls stay paired +;; with their results. +(def (agent-guardrails-step session-id provider effective content tcs round gr) + (let* ((cr (guardrails-check gr content tcs)) + (action (check-result-action cr))) + (cond + ;; Retry budget spent — stop and return what we have. + ((string=? action "fatal") + (log-warn logger "guardrails-fatal" `((reason . ,(check-result-reason cr)))) + (session-add-message session-id effective) + effective) + ;; Unusable response: unknown tool, or bare text under enforcement. + ((string=? action "retry") + (session-add-message session-id effective) + (cond + ((pair? tcs) + ;; Unknown tool: ride the corrective signal on the tool channel — + ;; one tool-error result per call — then re-infer (forge inference.py). + (for-each + (lambda (tc) + (session-add-message session-id + (make-tool-result (tool-call-id tc) + (string-append "[UnknownTool] " + (nudge-content (check-result-nudge cr)))))) + tcs) + (agent-loop session-id (session-get-messages session-id) (+ round 1) gr)) + (else + ;; Bare text under enforcement: fall back to a user-role nudge. + (session-add-message session-id + (make-user-message (nudge-content (check-result-nudge cr)))) + (agent-loop session-id (session-get-messages session-id) (+ round 1) gr)))) + ;; Hard round cap as a final safety net. + ((>= round *max-tool-rounds*) + (log-warn logger "max-rounds" `((round . ,round))) + (session-add-message session-id effective) + (let ((results (execute-tool-calls tcs))) + (for-each (lambda (r) (session-add-message session-id r)) results) + (let* ((final-msgs (refresh-system-prompt (session-get-messages session-id))) + (final (chat-with-expert provider final-msgs '()))) + (session-add-message session-id final) + final))) + ;; Execute: respond() ends the turn as plain text; otherwise run the + ;; validated calls and recurse. + (else + (let* ((calls (or (check-result-tool-calls cr) tcs)) + (rc (find-respond-call calls))) + (cond + (rc + (let ((final (make-assistant-message (respond-call->text rc) #f))) + (session-add-message session-id final) + final)) + (else + ;; If calls were rescued from bare text, effective is only the + ;; text — rebuild it carrying the tool_calls so results stay paired. + (let ((asst (if (null? tcs) (make-assistant-message #f calls) effective))) + (session-add-message session-id asst) + (let ((results (execute-tool-calls calls))) + (for-each (lambda (r) (session-add-message session-id r)) results) + (guardrails-record gr (map tool-call-name calls)) + (agent-loop session-id (session-get-messages session-id) + (+ round 1) gr)))))))))) + +(def (find-respond-call calls) + (cond + ((null? calls) #f) + ((respond-call? (car calls)) (car calls)) + (else (find-respond-call (cdr calls))))) -(def (agent-loop-stream session-id messages round) +(def (agent-loop-stream session-id messages round gr) ;; Streaming version: calls (current-stream-cb) for each text token. (let* ((provider (get-current-provider)) (tools (get-tool-schemas)) @@ -1143,26 +1228,76 @@ Be concise. Prefer edit over write for modifying existing files. (response (make-assistant-message effective-content (if (null? effective-tcs) #f effective-tcs)))) - (session-add-message session-id response) (cond - ((null? effective-tcs) response) - ((>= round *max-tool-rounds*) - (log-warn logger "max-rounds" `((round . ,round))) - (let ((results (execute-tool-calls effective-tcs))) - (for-each (lambda (r) (session-add-message session-id r)) results) - (let-values (((fc _tc _u) - (stream-chat-with-expert provider - (refresh-system-prompt (session-get-messages session-id)) - '() - (and raw-cb (make-tool-call-stream-filter raw-cb))))) - (let ((final (make-assistant-message - (if (string=? fc "") #f fc) #f))) - (session-add-message session-id final) - final)))) + ;; Bare text with respond-forcing off: a normal terminal answer. + ((and (null? effective-tcs) (not (forge-respond-enforced?))) + (session-add-message session-id response) + response) (else - (let ((results (execute-tool-calls effective-tcs))) - (for-each (lambda (r) (session-add-message session-id r)) results) - (agent-loop-stream session-id (session-get-messages session-id) (+ round 1))))))))) + (agent-guardrails-step-stream + session-id provider raw-cb response content effective-tcs round gr))))))) + +;; Streaming twin of agent-guardrails-step: same verdict logic, but the +;; round-cap finalization streams its tokens and recursion stays on the +;; streaming loop. +(def (agent-guardrails-step-stream session-id provider raw-cb response content tcs round gr) + (let* ((cr (guardrails-check gr content tcs)) + (action (check-result-action cr))) + (cond + ((string=? action "fatal") + (log-warn logger "guardrails-fatal" `((reason . ,(check-result-reason cr)))) + (session-add-message session-id response) + response) + ((string=? action "retry") + (session-add-message session-id response) + (cond + ((pair? tcs) + (for-each + (lambda (tc) + (session-add-message session-id + (make-tool-result (tool-call-id tc) + (string-append "[UnknownTool] " + (nudge-content (check-result-nudge cr)))))) + tcs) + (agent-loop-stream session-id (session-get-messages session-id) (+ round 1) gr)) + (else + (session-add-message session-id + (make-user-message (nudge-content (check-result-nudge cr)))) + (agent-loop-stream session-id (session-get-messages session-id) (+ round 1) gr)))) + ((>= round *max-tool-rounds*) + (log-warn logger "max-rounds" `((round . ,round))) + (session-add-message session-id response) + (let ((results (execute-tool-calls tcs))) + (for-each (lambda (r) (session-add-message session-id r)) results) + (let-values (((fc _tc _u) + (stream-chat-with-expert provider + (refresh-system-prompt (session-get-messages session-id)) + '() + (and raw-cb (make-tool-call-stream-filter raw-cb))))) + (let ((final (make-assistant-message (if (string=? fc "") #f fc) #f))) + (session-add-message session-id final) + final)))) + (else + (let* ((calls (or (check-result-tool-calls cr) tcs)) + (rc (find-respond-call calls))) + (cond + (rc + ;; Unwrap respond() — the model emitted a tool_call (suppressed by + ;; the stream filter), so push the answer through the cb now. + (let ((msg (respond-call->text rc))) + (when (and raw-cb (string? msg) (not (string=? msg ""))) + (raw-cb msg)) + (let ((final (make-assistant-message msg #f))) + (session-add-message session-id final) + final))) + (else + (let ((asst (if (null? tcs) (make-assistant-message #f calls) response))) + (session-add-message session-id asst) + (let ((results (execute-tool-calls calls))) + (for-each (lambda (r) (session-add-message session-id r)) results) + (guardrails-record gr (map tool-call-name calls)) + (agent-loop-stream session-id (session-get-messages session-id) + (+ round 1) gr)))))))))) (def (execute-tool-calls tool-calls) (log-info logger "executing-tools" `((count . ,(length tool-calls)))) new file mode 100644 --- /dev/null +++ b/src/jcode/core/compaction-strategy.ss @@ -0,0 +1,259 @@ +;;; jcode compaction strategies +;;; +;;; Faithful port of forge's context/strategies.py: NoCompact, +;;; SlidingWindowCompact, TieredCompact — plus JcodeLegacy, which wraps the +;;; pre-forge compaction.ss (a sliding-window + in-place-truncate hybrid) so +;;; the old behaviour stays one config flag away. The default is Tiered. +;;; +;;; A strategy is a procedure (strategy messages budget-tokens) that returns +;;; a pair (compacted-messages . phase-reached). phase 0 = untouched; 1+ is +;;; how aggressively it compacted (Tiered defines 1/2/3, see below). +;;; +;;; forge tags every Message with a MessageType and a step_index in metadata. +;;; jcode's message struct carries neither, so we DERIVE both: +;;; * message-derived-type — from role + tool-calls/tool-call-id/thinking +;;; * derive-step-indices — one "iteration" (step) opens at each assistant +;;; or user message; trailing tool results inherit it (matches forge's +;;; "iteration = one assistant message + N tool results"). +;;; The nudge MessageTypes (step/prerequisite/retry) are NOT shape-derivable — +;;; jcode injects nudges as plain user messages — so Phase-1 nudge-dropping is +;;; effectively a no-op until the message struct grows an explicit type tag. +;;; Tool-result truncation/dropping and reasoning/text dropping (the bulk of +;;; the win) work fully. + +(export message-derived-type + derive-step-indices + strategy-estimate-tokens + find-eligible-end + make-no-compact + make-sliding-window + make-tiered + make-jcode-legacy + compaction-strategy-by-name + configured-compaction-strategy-name + run-configured-compaction) + +(import :jcode/core/message + :jcode/guardrails/message-type + :jcode/core/config + :jcode/core/log + :jcode/core/compaction) + +(def logger (make-logger "compaction-strategy")) + +(def TRUNCATE-CHARS 200) + +;; ── Derivation: MessageType and step_index from jcode message shape ── + +(def (message-derived-type msg) + "Classify a jcode message into a forge MessageType tag by its shape." + (let ((role (message-role msg))) + (cond + ((equal? role "system") message-type-system-prompt) + ((equal? role "tool") message-type-tool-result) + ((equal? role "user") message-type-user-input) + ((equal? role "assistant") + (let ((tcs (message-tool-calls msg))) + (cond + ;; A message carrying tool calls is a tool_call message — its + ;; incidental text rides along (forge splits these; jcode bundles). + ((and tcs (pair? tcs)) message-type-tool-call) + ((and (message-thinking msg) + (let ((c (message-content msg))) + (or (not c) (= 0 (string-length c))))) + message-type-reasoning) + (else message-type-text-response)))) + (else message-type-text-response)))) + +(def (derive-step-indices messages) + "Parallel list of step-index (or #f) for MESSAGES. messages[0]/[1] are the + protected header → #f. From index 2, each assistant/user message opens a + new iteration; tool results inherit the current open step." + (let loop ((ms messages) (i 0) (step 0) (acc '())) + (cond + ((null? ms) (reverse acc)) + ((< i 2) (loop (cdr ms) (+ i 1) step (cons #f acc))) + (else + (let ((role (message-role (car ms)))) + (cond + ((or (equal? role "assistant") (equal? role "user")) + (let ((s (+ step 1))) (loop (cdr ms) (+ i 1) s (cons s acc)))) + (else + (let ((s (if (= step 0) 1 step))) + (loop (cdr ms) (+ i 1) step (cons s acc)))))))))) + +;; ── Shared helpers ────────────────────────────────────────────────── + +(def (strategy-estimate-tokens messages) + ;; forge _estimate_tokens: sum of content lengths // 4 (content only). + (quotient + (apply + (map (lambda (m) (string-length (or (message-content m) ""))) messages)) + 4)) + +(def (distinct-consecutive xs) + ;; Append a value whenever it differs from the previously appended one; + ;; skip #f. Mirrors forge's seen_steps accumulation exactly. + (let loop ((xs xs) (last #f) (acc '())) + (cond + ((null? xs) (reverse acc)) + ((not (car xs)) (loop (cdr xs) last acc)) + ((eqv? (car xs) last) (loop (cdr xs) last acc)) + (else (loop (cdr xs) (car xs) (cons (car xs) acc)))))) + +(def (find-eligible-end messages keep-recent) + "Boundary index: messages before it are eligible for compaction. Protects + the last KEEP-RECENT iterations (and always messages[0]/[1])." + (let* ((steps (derive-step-indices messages)) + (seen (distinct-consecutive steps)) + (total (length messages))) + (cond + ((<= (length seen) keep-recent) 2) + (else + (let ((cutoff (list-ref seen (- (length seen) keep-recent)))) + (let loop ((xs steps) (i 0)) + (cond + ((null? xs) total) + ((and (>= i 2) (car xs) (>= (car xs) cutoff)) i) + (else (loop (cdr xs) (+ i 1)))))))))) + +(def (nudge-type? ty) + (or (equal? ty message-type-step-nudge) + (equal? ty message-type-prerequisite-nudge) + (equal? ty message-type-retry-nudge))) + +;; ── NoCompact ─────────────────────────────────────────────────────── + +(def (make-no-compact) + "Passthrough. Returns messages unchanged (phase 0)." + (lambda (messages budget) (cons messages 0))) + +;; ── SlidingWindowCompact ──────────────────────────────────────────── + +(def (make-sliding-window keep-recent threshold) + "Keep system + first user + the last KEEP-RECENT iterations; drop the + middle. Fires only above THRESHOLD fraction of budget." + (lambda (messages budget) + (let ((trigger (exact (floor (* budget threshold))))) + (if (< (strategy-estimate-tokens messages) trigger) + (cons messages 0) + (let ((ee (find-eligible-end messages keep-recent))) + (if (<= ee 2) + (cons messages 1) + (cons (append (list (car messages) (cadr messages)) + (list-tail messages ee)) + 1))))))) + +;; ── TieredCompact (three-phase, default) ──────────────────────────── + +(def (make-tiered keep-recent threshold phase-thresholds) + "Three-phase compaction. PHASE-THRESHOLDS is #f (use THRESHOLD for all + three) or a list (p1 p2 p3) of budget fractions. Phases: + 1. drop nudges; truncate tool_results to first 200 chars + 2. + drop tool_results entirely (reasoning + text preserved) + 3. + drop reasoning + text_response (tool_call skeleton only)